diff --git a/api/openai/request.go b/api/openai/request.go index 20413115..2e7bdc76 100644 --- a/api/openai/request.go +++ b/api/openai/request.go @@ -27,7 +27,7 @@ func readInput(c *fiber.Ctx, o *options.Option, randomModel bool) (string, *sche input.Cancel = cancel // Get input data from the request body if err := c.BodyParser(input); err != nil { - return "", nil, err + return "", nil, fmt.Errorf("failed parsing request body: %w", err) } modelFile := input.Model @@ -165,7 +165,7 @@ func updateConfig(config *config.Config, input *schema.OpenAIRequest) { // Decode each request's message content index := 0 - for _, m := range input.Messages { + for i, m := range input.Messages { switch content := m.Content.(type) { case string: m.StringContent = content @@ -175,14 +175,14 @@ func updateConfig(config *config.Config, input *schema.OpenAIRequest) { json.Unmarshal(dat, &c) for _, pp := range c { if pp.Type == "text" { - m.StringContent = pp.Text + input.Messages[i].StringContent = pp.Text } else if pp.Type == "image_url" { // Detect if pp.ImageURL is an URL, if it is download the image and encode it in base64: - base64, err := getBase64Image(pp.ImageURL) + base64, err := getBase64Image(pp.ImageURL.URL) if err == nil { - m.StringImages = append(m.StringImages, base64) // TODO: make sure that we only return base64 stuff + input.Messages[i].StringImages = append(input.Messages[i].StringImages, base64) // TODO: make sure that we only return base64 stuff // set a placeholder for each image - m.StringContent = m.StringContent + fmt.Sprintf("[img-%d]", index) + input.Messages[i].StringContent = input.Messages[i].StringContent + fmt.Sprintf("[img-%d]", index) index++ } else { fmt.Print("Failed encoding image", err) diff --git a/api/schema/openai.go b/api/schema/openai.go index 6d276363..899f1b74 100644 --- a/api/schema/openai.go +++ b/api/schema/openai.go @@ -56,9 +56,13 @@ type Choice struct { } type Content struct { - Type string `json:"type" yaml:"type"` - Text string `json:"text" yaml:"text"` - ImageURL string `json:"image_url" yaml:"image_url"` + Type string `json:"type" yaml:"type"` + Text string `json:"text" yaml:"text"` + ImageURL ContentURL `json:"image_url" yaml:"image_url"` +} + +type ContentURL struct { + URL string `json:"url" yaml:"url"` } type Message struct { diff --git a/backend/cpp/llama/grpc-server.cpp b/backend/cpp/llama/grpc-server.cpp index 311f2018..6d5e3f23 100644 --- a/backend/cpp/llama/grpc-server.cpp +++ b/backend/cpp/llama/grpc-server.cpp @@ -1974,7 +1974,7 @@ static void params_parse(const backend::ModelOptions* request, if (!request->mmproj().empty()) { // get the directory of modelfile std::string model_dir = params.model.substr(0, params.model.find_last_of("/\\")); - params.mmproj = model_dir + request->mmproj(); + params.mmproj = model_dir + "/"+ request->mmproj(); } // params.model_alias ?? params.model_alias = request->modelfile();