feat(llama.cpp): guess model defaults from file (#2522)

* wip: guess informations from gguf file

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* update go mod

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Small fixups

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Identify llama3

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Do not try to guess the name, as reading gguf files can be expensive

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Allow to disable guessing

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/cli/transcript.go

@@ -30,7 +30,7 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error {
 		AssetsDestination: t.BackendAssetsPath,
 	}
 
-	cl := config.NewBackendConfigLoader()
+	cl := config.NewBackendConfigLoader(t.ModelsPath)
 	ml := model.NewModelLoader(opts.ModelPath)
 	if err := cl.LoadBackendConfigsFromPath(t.ModelsPath); err != nil {
 		return err

core/config/application_config.go

@@ -306,6 +306,7 @@ func (o *ApplicationConfig) ToConfigLoaderOptions() []ConfigLoaderOption {
 		LoadOptionDebug(o.Debug),
 		LoadOptionF16(o.F16),
 		LoadOptionThreads(o.Threads),
+		ModelPath(o.ModelPath),
 	}
 }
 

core/config/backend_config.go

@@ -21,7 +21,7 @@ type TTSConfig struct {
 	Voice string `yaml:"voice"`
 
 	// Vall-e-x
-	VallE    VallE  `yaml:"vall-e"`
+	VallE VallE `yaml:"vall-e"`
 }
 
 type BackendConfig struct {
@@ -368,6 +368,8 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
 	if debug {
 		cfg.Debug = &trueV
 	}
+
+	guessDefaultsFromFile(cfg, lo.modelPath)
 }
 
 func (c *BackendConfig) Validate() bool {
@@ -400,3 +402,7 @@ func (c *BackendConfig) Validate() bool {
 
 	return true
 }
+
+func (c *BackendConfig) HasTemplate() bool {
+	return c.TemplateConfig.Completion != "" || c.TemplateConfig.Edit != "" || c.TemplateConfig.Chat != "" || c.TemplateConfig.ChatMessage != ""
+}

core/config/backend_config_loader.go

@@ -19,17 +19,20 @@ import (
 )
 
 type BackendConfigLoader struct {
-	configs map[string]BackendConfig
+	configs   map[string]BackendConfig
+	modelPath string
 	sync.Mutex
 }
 
-func NewBackendConfigLoader() *BackendConfigLoader {
+func NewBackendConfigLoader(modelPath string) *BackendConfigLoader {
 	return &BackendConfigLoader{
-		configs: make(map[string]BackendConfig),
+		configs:   make(map[string]BackendConfig),
+		modelPath: modelPath,
 	}
 }
 
 type LoadOptions struct {
+	modelPath        string
 	debug            bool
 	threads, ctxSize int
 	f16              bool
@@ -53,6 +56,12 @@ func LoadOptionContextSize(ctxSize int) ConfigLoaderOption {
 	}
 }
 
+func ModelPath(modelPath string) ConfigLoaderOption {
+	return func(o *LoadOptions) {
+		o.modelPath = modelPath
+	}
+}
+
 func LoadOptionF16(f16 bool) ConfigLoaderOption {
 	return func(o *LoadOptions) {
 		o.f16 = f16

core/config/config_test.go

@@ -26,7 +26,7 @@ var _ = Describe("Test cases for config related functions", func() {
 
 		It("Test LoadConfigs", func() {
 
-			bcl := NewBackendConfigLoader()
+			bcl := NewBackendConfigLoader(os.Getenv("MODELS_PATH"))
 			err := bcl.LoadBackendConfigsFromPath(os.Getenv("MODELS_PATH"))
 
 			Expect(err).To(BeNil())

core/config/guesser.go

@@ -0,0 +1,87 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/rs/zerolog/log"
+
+	gguf "github.com/thxcode/gguf-parser-go"
+)
+
+type familyType uint8
+
+const (
+	Unknown familyType = iota
+	LLaMa3             = iota
+	LLama2             = iota
+)
+
+var defaultsTemplate map[familyType]TemplateConfig = map[familyType]TemplateConfig{
+	LLaMa3: {
+		Chat:        "<|begin_of_text|>{{.Input }}\n<|start_header_id|>assistant<|end_header_id|>",
+		ChatMessage: "<|start_header_id|>{{ .RoleName }}<|end_header_id|>\n\n{{.Content }}<|eot_id|>",
+	},
+}
+
+func guessDefaultsFromFile(cfg *BackendConfig, modelPath string) {
+
+	if os.Getenv("LOCALAI_DISABLE_GUESSING") == "true" {
+		log.Debug().Msgf("guessDefaultsFromFile: %s", "guessing disabled with LOCALAI_DISABLE_GUESSING")
+		return
+	}
+
+	if modelPath == "" {
+		log.Debug().Msgf("guessDefaultsFromFile: %s", "modelPath is empty")
+		return
+	}
+
+	if cfg.HasTemplate() {
+		// nothing to guess here
+		log.Debug().Any("name", cfg.Name).Msgf("guessDefaultsFromFile: %s", "template already set")
+		return
+	}
+
+	// We try to guess only if we don't have a template defined already
+	f, err := gguf.ParseGGUFFile(filepath.Join(modelPath, cfg.ModelFileName()))
+	if err != nil {
+		// Only valid for gguf files
+		log.Debug().Msgf("guessDefaultsFromFile: %s", "not a GGUF file")
+		return
+	}
+
+	log.Debug().
+		Any("eosTokenID", f.Tokenizer().EOSTokenID).
+		Any("bosTokenID", f.Tokenizer().BOSTokenID).
+		Any("modelName", f.Model().Name).
+		Any("architecture", f.Architecture().Architecture).Msgf("Model file loaded: %s", cfg.ModelFileName())
+
+	// guess the name
+	if cfg.Name == "" {
+		cfg.Name = f.Model().Name
+	}
+
+	family := identifyFamily(f)
+
+	if family == Unknown {
+		log.Debug().Msgf("guessDefaultsFromFile: %s", "family not identified")
+		return
+	}
+
+	templ, ok := defaultsTemplate[family]
+	if ok {
+		cfg.TemplateConfig = templ
+		log.Debug().Any("family", family).Msgf("guessDefaultsFromFile: guessed template %+v", cfg.TemplateConfig)
+	} else {
+		log.Debug().Any("family", family).Msgf("guessDefaultsFromFile: no template found for family")
+	}
+}
+
+func identifyFamily(f *gguf.GGUFFile) familyType {
+	switch {
+	case f.Architecture().Architecture == "llama" && f.Tokenizer().EOSTokenID == 128009:
+		return LLaMa3
+	}
+
+	return Unknown
+}

core/http/endpoints/openai/request.go

@@ -289,6 +289,7 @@ func mergeRequestWithConfig(modelFile string, input *schema.OpenAIRequest, cm *c
 		config.LoadOptionThreads(threads),
 		config.LoadOptionContextSize(ctx),
 		config.LoadOptionF16(f16),
+		config.ModelPath(loader.ModelPath),
 	)
 
 	// Set the parameters for the language model prediction

core/startup/startup.go

@@ -62,7 +62,7 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
 	//
 	pkgStartup.PreloadModelsConfigurations(options.ModelLibraryURL, options.ModelPath, options.ModelsURL...)
 
-	cl := config.NewBackendConfigLoader()
+	cl := config.NewBackendConfigLoader(options.ModelPath)
 	ml := model.NewModelLoader(options.ModelPath)
 
 	configLoaderOpts := options.ToConfigLoaderOptions()
@@ -151,7 +151,7 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
 func createApplication(appConfig *config.ApplicationConfig) *core.Application {
 	app := &core.Application{
 		ApplicationConfig:   appConfig,
-		BackendConfigLoader: config.NewBackendConfigLoader(),
+		BackendConfigLoader: config.NewBackendConfigLoader(appConfig.ModelPath),
 		ModelLoader:         model.NewModelLoader(appConfig.ModelPath),
 	}