models(gallery): add mistral-0.3 and command-r, update functions (#2388)

* models(gallery): add mistral-0.3 and command-r, update functions

Add also disable_parallel_new_lines to disable newlines in the JSON
output when forcing parallel tools. Some models (like mistral) might be
very sensible to that when being used for function calling.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* models(gallery): add aya-23-8b

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

pkg/functions/grammar_json_schema.go

@@ -50,6 +50,9 @@ var (
 			[^"\\] |
 			"\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
 		  )* "\"" space`,
+		// TODO: we shouldn't forbid \" and \\ or all unicode and have this branch here,
+		// however, if we don't have it, the grammar will be ambiguous and
+		// empirically results are way worse.
 		"freestring": `(
 			[^"\\] |
 			"\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
@@ -111,12 +114,18 @@ func (sc *JSONSchemaConverter) addRule(name, rule string) string {
 	return key
 }
 
-const array = `arr  ::=
+const arrayNewLines = `arr  ::=
   "[\n"  (
 		realvalue
     (",\n"  realvalue)*
   )? "]"`
 
+const array = `arr  ::=
+  "["  (
+		realvalue
+    (","  realvalue)*
+  )? "]"`
+
 func (sc *JSONSchemaConverter) finalizeGrammar(options ...func(*GrammarOption)) string {
 
 	grammarOpts := &GrammarOption{}
@@ -124,6 +133,7 @@ func (sc *JSONSchemaConverter) finalizeGrammar(options ...func(*GrammarOption))
 
 	suffix := grammarOpts.Suffix
 	maybeArray := grammarOpts.MaybeArray
+	disableParallelNewLines := grammarOpts.DisableParallelNewLines
 	maybeString := grammarOpts.MaybeString
 	noMixedFreeString := grammarOpts.NoMixedFreeString
 
@@ -177,7 +187,11 @@ func (sc *JSONSchemaConverter) finalizeGrammar(options ...func(*GrammarOption))
 	}
 
 	lines = append(lines, fmt.Sprintf("%s ::= %s", "root", newRoot))
-	lines = append(lines, array)
+	if disableParallelNewLines {
+		lines = append(lines, array)
+	} else {
+		lines = append(lines, arrayNewLines)
+	}
 
 	if maybeArray {
 		lines = append(lines, `mixedstring ::= freestring | freestring arr | freestring realvalue | realvalue | arr`)

pkg/functions/grammar_json_schema_test.go

@@ -427,5 +427,22 @@ var _ = Describe("JSON schema grammar tests", func() {
 			}
 			Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar)
 		})
+
+		It("generates parallel tools without newlines in JSON", func() {
+			structuredGrammar := JSONFunctionStructureName{
+				OneOf: testFunctionsName}
+			content := `arr  ::=
+"["  (
+realvalue
+(","  realvalue)*
+)? "]"`
+			grammar := structuredGrammar.Grammar(functions.EnableMaybeString, functions.EnableMaybeArray, functions.DisableParallelNewLines)
+			results := strings.Split(content, "\n")
+			for _, r := range results {
+				if r != "" {
+					Expect(grammar).To(ContainSubstring(r))
+				}
+			}
+		})
 	})
 })

pkg/functions/options.go

@@ -1,11 +1,12 @@
 package functions
 
 type GrammarOption struct {
-	PropOrder         string
-	Suffix            string
-	MaybeArray        bool
-	MaybeString       bool
-	NoMixedFreeString bool
+	PropOrder               string
+	Suffix                  string
+	MaybeArray              bool
+	DisableParallelNewLines bool
+	MaybeString             bool
+	NoMixedFreeString       bool
 }
 
 func (o *GrammarOption) Apply(options ...func(*GrammarOption)) {
@@ -18,6 +19,10 @@ var EnableMaybeArray = func(o *GrammarOption) {
 	o.MaybeArray = true
 }
 
+var DisableParallelNewLines = func(o *GrammarOption) {
+	o.DisableParallelNewLines = true
+}
+
 var EnableMaybeString = func(o *GrammarOption) {
 	o.MaybeString = true
 }

pkg/functions/parse.go

@@ -12,6 +12,8 @@ type GrammarConfig struct {
 	// ParallelCalls enables the LLM to return multiple function calls in the same response
 	ParallelCalls bool `yaml:"parallel_calls"`
 
+	DisableParallelNewLines bool `yaml:"disable_parallel_new_lines"`
+
 	// MixedMode enables the LLM to return strings and not only JSON objects
 	// This is useful for models to not constraing returning only JSON and also messages back to the user
 	MixedMode bool `yaml:"mixed_mode"`
@@ -81,6 +83,9 @@ func (g GrammarConfig) Options() []func(o *GrammarOption) {
 	if g.ParallelCalls {
 		opts = append(opts, EnableMaybeArray)
 	}
+	if g.DisableParallelNewLines {
+		opts = append(opts, DisableParallelNewLines)
+	}
 	if g.Prefix != "" {
 		opts = append(opts, SetPrefix(g.Prefix))
 	}
@@ -134,7 +139,7 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC
 			var singleObj map[string]interface{}
 			err = json.Unmarshal([]byte(s), &singleObj)
 			if err != nil {
-				log.Warn().Err(err).Str("escapedLLMResult", s).Msg("unable to unmarshal llm result")
+				log.Debug().Err(err).Str("escapedLLMResult", s).Msg("unable to unmarshal llm result in a single object or an array of JSON objects")
 			} else {
 				ss = []map[string]interface{}{singleObj}
 			}
@@ -177,6 +182,7 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC
 			match := respRegex.FindStringSubmatch(llmresult)
 			if len(match) >= 2 {
 				llmresult = match[1]
+				log.Debug().Msgf("LLM result(JSONRegexMatch): %s", llmresult)
 				break
 			}
 		}