dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-06 02:45:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 12

feat(ui): add audio upload button in chat view

Browse source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto 2025-05-29 22:42:08 +02:00
parent 0870bf5af6
commit dbee564f28
5 changed files with 201 additions and 180 deletions
Download patch file Download diff file Expand all files Collapse all files

94
core/http/static/chat.js
View file

@ -49,12 +49,13 @@ function submitSystemPrompt(event) {
}
var image = "";
var audio = "";
function submitPrompt(event) {
event.preventDefault();
const input = document.getElementById("input").value;
Alpine.store("chat").add("user", input, image);
Alpine.store("chat").add("user", input, image, audio);
document.getElementById("input").value = "";
const systemPrompt = localStorage.getItem("system_prompt");
Alpine.nextTick(() => { document.getElementById('messages').scrollIntoView(false); });
@ -62,7 +63,6 @@ function submitPrompt(event) {
}
function readInputImage() {
if (!this.files || !this.files[0]) return;
const FR = new FileReader();
@ -74,15 +74,25 @@ function readInputImage() {
FR.readAsDataURL(this.files[0]);
}
function readInputAudio() {
if (!this.files || !this.files[0]) return;
async function promptGPT(systemPrompt, input) {
const FR = new FileReader();
FR.addEventListener("load", function(evt) {
audio = evt.target.result;
});
FR.readAsDataURL(this.files[0]);
}
async function promptGPT(systemPrompt, input) {
const model = document.getElementById("chat-model").value;
// Set class "loader" to the element with "loader" id
//document.getElementById("loader").classList.add("loader");
// Make the "loader" visible
toggleLoader(true);
messages = Alpine.store("chat").messages();
// if systemPrompt isn't empty, push it at the start of messages
@ -93,9 +103,9 @@ function readInputImage() {
});
}
// loop all messages, and check if there are images. If there are, we need to change the content field
// loop all messages, and check if there are images or audios. If there are, we need to change the content field
messages.forEach((message) => {
if (message.image) {
if (message.image || message.audio) {
// The content field now becomes an array
message.content = [
{
@ -103,6 +113,8 @@ function readInputImage() {
"text": message.content
}
]
if (message.image) {
message.content.push(
{
"type": "image_url",
@ -111,44 +123,30 @@ function readInputImage() {
}
}
);
// remove the image field
delete message.image;
}
if (message.audio) {
message.content.push(
{
"type": "audio_url",
"audio_url": {
"url": message.audio,
}
}
);
delete message.audio;
}
}
});
// reset the form and the image
// reset the form and the files
image = "";
audio = "";
document.getElementById("input_image").value = null;
document.getElementById("input_audio").value = null;
document.getElementById("fileName").innerHTML = "";
// if (image) {
// // take the last element content's and add the image
// last_message = messages[messages.length - 1]
// // The content field now becomes an array
// last_message.content = [
// {
// "type": "text",
// "text": last_message.content
// }
// ]
// last_message.content.push(
// {
// "type": "image_url",
// "image_url": {
// "url": image,
// }
// }
// );
// // and we replace it in the messages array
// messages[messages.length - 1] = last_message
// // reset the form and the image
// image = "";
// document.getElementById("input_image").value = null;
// document.getElementById("fileName").innerHTML = "";
// }
// Source: https://stackoverflow.com/a/75751803/11386095
const response = await fetch("v1/chat/completions", {
method: "POST",
@ -256,23 +254,23 @@ function readInputImage() {
document.getElementById('messages').scrollIntoView(false)
// set focus to the input
document.getElementById("input").focus();
}
}
document.getElementById("system_prompt").addEventListener("submit", submitSystemPrompt);
document.getElementById("system_prompt").addEventListener("submit", submitSystemPrompt);
document.getElementById("prompt").addEventListener("submit", submitPrompt);
document.getElementById("input").focus();
document.getElementById("input_image").addEventListener("change", readInputImage);
document.getElementById("input_audio").addEventListener("change", readInputAudio);
document.getElementById("prompt").addEventListener("submit", submitPrompt);
document.getElementById("input").focus();
document.getElementById("input_image").addEventListener("change", readInputImage);
storesystemPrompt = localStorage.getItem("system_prompt");
if (storesystemPrompt) {
storesystemPrompt = localStorage.getItem("system_prompt");
if (storesystemPrompt) {
document.getElementById("systemPrompt").value = storesystemPrompt;
} else {
} else {
document.getElementById("systemPrompt").value = null;
}
}
marked.setOptions({
marked.setOptions({
highlight: function (code) {
return hljs.highlightAuto(code).value;
},
});
});

20
core/http/views/chat.html
View file

@ -218,6 +218,8 @@ SOFTWARE.
Start chatting with the AI by typing a prompt in the input field below and pressing Enter.
For models that support images, you can upload an image by clicking the paperclip
<i class="fa-solid fa-paperclip"></i> icon.
For models that support audio, you can upload an audio file by clicking the microphone
<i class="fa-solid fa-microphone"></i> icon.
</p>
<div id="messages" class="max-w-3xl mx-auto">
<template x-for="message in history">
@ -290,6 +292,12 @@ SOFTWARE.
class="fa-solid fa-paperclip text-gray-400 absolute right-12 top-4 text-lg p-2 hover:text-blue-400 transition-colors duration-200"
title="Attach an image"
></button>
<button
type="button"
onclick="document.getElementById('input_audio').click()"
class="fa-solid fa-microphone text-gray-400 absolute right-20 top-4 text-lg p-2 hover:text-blue-400 transition-colors duration-200"
title="Attach an audio file"
></button>
<!-- Send button and loader in the same position -->
<div class="absolute right-3 top-4">
@ -320,6 +328,13 @@ SOFTWARE.
style="display: none;"
@change="fileName = $event.target.files[0].name"
/>
<input
id="input_audio"
type="file"
accept="audio/*"
style="display: none;"
@change="fileName = $event.target.files[0].name"
/>
</div>
</form>
</div>
@ -381,7 +396,7 @@ SOFTWARE.
clear() {
this.history.length = 0;
},
add(role, content, image) {
add(role, content, image, audio) {
const N = this.history.length - 1;
if (this.history.length && this.history[N].role === role) {
this.history[N].content += content;
@ -394,7 +409,7 @@ SOFTWARE.
lines.forEach((line) => {
c += DOMPurify.sanitize(marked.parse(line));
});
this.history.push({ role, content, html: c, image });
this.history.push({ role, content, html: c, image, audio });
}
document.getElementById('messages').scrollIntoView(false);
const parser = new DOMParser();
@ -418,6 +433,7 @@ SOFTWARE.
role: message.role,
content: message.content,
image: message.image,
audio: message.audio,
}));
},
});

1
go.mod
View file

@ -82,6 +82,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e // indirect
github.com/pion/datachannel v1.5.10 // indirect
github.com/pion/dtls/v2 v2.2.12 // indirect
github.com/pion/dtls/v3 v3.0.4 // indirect

2
go.sum
View file

@ -546,6 +546,8 @@ github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw=
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0=
github.com/otiai10/mint v1.6.1 h1:kgbTJmOpp/0ce7hk3H8jiSuR0MXmpwWRfqUdKww17qg=
github.com/otiai10/mint v1.6.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
github.com/otiai10/openaigo v1.7.0 h1:AOQcOjRRM57ABvz+aI2oJA/Qsz1AydKbdZAlGiKyCqg=

18
pkg/utils/base64.go
View file

@ -5,14 +5,19 @@ import (
"fmt"
"io"
"net/http"
"regexp"
"strings"
"time"
"github.com/rs/zerolog/log"
)
var base64DownloadClient http.Client = http.Client{
Timeout: 30 * time.Second,
}
var dataURIPattern = regexp.MustCompile(`^data:([^;]+);base64,`)
// GetContentURIAsBase64 checks if the string is an URL, if it's an URL downloads the content in memory encodes it in base64 and returns the base64 string, otherwise returns the string by stripping base64 data headers
func GetContentURIAsBase64(s string) (string, error) {
if strings.HasPrefix(s, "http") {
@ -36,12 +41,11 @@ func GetContentURIAsBase64(s string) (string, error) {
return encoded, nil
}
// if the string instead is prefixed with "data:image/...;base64,", drop it
dropPrefix := []string{"data:image/jpeg;base64,", "data:image/png;base64,"}
for _, prefix := range dropPrefix {
if strings.HasPrefix(s, prefix) {
return strings.ReplaceAll(s, prefix, ""), nil
// Match any data URI prefix pattern
if match := dataURIPattern.FindString(s); match != "" {
log.Debug().Msgf("Found data URI prefix: %s", match)
return strings.Replace(s, match, "", 1), nil
}
}
return "", fmt.Errorf("not valid string")
return "", fmt.Errorf("not valid base64 data type string")
}