switching workstations, stash the merge

apiv2/config.go

@@ -31,12 +31,6 @@ type SpecificConfig[RequestModel any] struct {
 	RequestDefaults RequestModel `yaml:"request_defaults" mapstructure:"request_defaults"`
 }
 
-// type Config struct {
-// 	Registration    ConfigRegistration `yaml:"registration"`
-// 	LocalPaths      ConfigLocalPaths   `yaml:"local_paths"`
-// 	RequestDefaults interface{}        `yaml:"request_defaults"`
-// }
-
 type Config interface {
 	GetRequestDefaults() interface{}
 	GetLocalPaths() ConfigLocalPaths
@@ -198,212 +192,28 @@ func (cm *ConfigManager) ListConfigs() []ConfigRegistration {
 	return cm.listConfigs()
 }
 
-// // Not sure about this one, but it seems like a decent place to stick it for an experiment at least.
+// These functions I'm a bit dubious about. I think there's a better refactoring down in pkg/model
-// func (cm *ConfigManager) GetTextConfigForRequest()
+// But to get a minimal test up and running, here we go!
 
-// func (cm *ConfigMerger) LoadConfigs(path string) error {
+func (sc SpecificConfig[RequestModel]) ToModelOptions() []llama.ModelOption {
-// 	cm.Lock()
-// 	defer cm.Unlock()
-// 	files, err := ioutil.ReadDir(path)
-// 	if err != nil {
-// 		return err
-// 	}
 
-// 	for _, file := range files {
+	llamaOpts := []llama.ModelOption{}
-// 		// Skip templates, YAML and .keep files
-// 		if !strings.Contains(file.Name(), ".yaml") {
-// 			continue
-// 		}
-// 		c, err := ReadConfig(filepath.Join(path, file.Name()))
-// 		if err == nil {
-// 			cm.configs[ConfigLookup{Name: c.Name, Endpoint: c.Endpoint}] = *c
-// 		}
-// 	}
 
-// 	return nil
+	// Code to Port:	
-// }
 
-// func (cm *ConfigMerger) Get
+	// if c.ContextSize != 0 {
+	// 	llamaOpts = append(llamaOpts, llama.SetContext(c.ContextSize))
+	// }
+	// if c.F16 {
+	// 	llamaOpts = append(llamaOpts, llama.EnableF16Memory)
+	// }
+	// if c.Embeddings {
+	// 	llamaOpts = append(llamaOpts, llama.EnableEmbeddings)
+	// }
 
-// func updateConfig(config *Config, input *OpenAIRequest) {
+	// if c.NGPULayers != 0 {
-// 	if input.Echo {
+	// 	llamaOpts = append(llamaOpts, llama.SetGPULayers(c.NGPULayers))
-// 		config.Echo = input.Echo
+	// }
-// 	}
-// 	if input.TopK != 0 {
-// 		config.TopK = input.TopK
-// 	}
-// 	if input.TopP != 0 {
-// 		config.TopP = input.TopP
-// 	}
 
-// 	if input.Temperature != 0 {
+	return llamaOpts
-// 		config.Temperature = input.Temperature
+}
-// 	}
-
-// 	if input.Maxtokens != 0 {
-// 		config.Maxtokens = input.Maxtokens
-// 	}
-
-// 	switch stop := input.Stop.(type) {
-// 	case string:
-// 		if stop != "" {
-// 			config.StopWords = append(config.StopWords, stop)
-// 		}
-// 	case []interface{}:
-// 		for _, pp := range stop {
-// 			if s, ok := pp.(string); ok {
-// 				config.StopWords = append(config.StopWords, s)
-// 			}
-// 		}
-// 	}
-
-// 	if input.RepeatPenalty != 0 {
-// 		config.RepeatPenalty = input.RepeatPenalty
-// 	}
-
-// 	if input.Keep != 0 {
-// 		config.Keep = input.Keep
-// 	}
-
-// 	if input.Batch != 0 {
-// 		config.Batch = input.Batch
-// 	}
-
-// 	if input.F16 {
-// 		config.F16 = input.F16
-// 	}
-
-// 	if input.IgnoreEOS {
-// 		config.IgnoreEOS = input.IgnoreEOS
-// 	}
-
-// 	if input.Seed != 0 {
-// 		config.Seed = input.Seed
-// 	}
-
-// 	if input.Mirostat != 0 {
-// 		config.Mirostat = input.Mirostat
-// 	}
-
-// 	if input.MirostatETA != 0 {
-// 		config.MirostatETA = input.MirostatETA
-// 	}
-
-// 	if input.MirostatTAU != 0 {
-// 		config.MirostatTAU = input.MirostatTAU
-// 	}
-
-// 	switch inputs := input.Input.(type) {
-// 	case string:
-// 		if inputs != "" {
-// 			config.InputStrings = append(config.InputStrings, inputs)
-// 		}
-// 	case []interface{}:
-// 		for _, pp := range inputs {
-// 			switch i := pp.(type) {
-// 			case string:
-// 				config.InputStrings = append(config.InputStrings, i)
-// 			case []interface{}:
-// 				tokens := []int{}
-// 				for _, ii := range i {
-// 					tokens = append(tokens, int(ii.(float64)))
-// 				}
-// 				config.InputToken = append(config.InputToken, tokens)
-// 			}
-// 		}
-// 	}
-
-// 	switch p := input.Prompt.(type) {
-// 	case string:
-// 		config.PromptStrings = append(config.PromptStrings, p)
-// 	case []interface{}:
-// 		for _, pp := range p {
-// 			if s, ok := pp.(string); ok {
-// 				config.PromptStrings = append(config.PromptStrings, s)
-// 			}
-// 		}
-// 	}
-// }
-// func readInput(c *fiber.Ctx, loader *model.ModelLoader, randomModel bool) (string, *OpenAIRequest, error) {
-// 	input := new(OpenAIRequest)
-// 	// Get input data from the request body
-// 	if err := c.BodyParser(input); err != nil {
-// 		return "", nil, err
-// 	}
-
-// 	modelFile := input.Model
-
-// 	if c.Params("model") != "" {
-// 		modelFile = c.Params("model")
-// 	}
-
-// 	received, _ := json.Marshal(input)
-
-// 	log.Debug().Msgf("Request received: %s", string(received))
-
-// 	// Set model from bearer token, if available
-// 	bearer := strings.TrimLeft(c.Get("authorization"), "Bearer ")
-// 	bearerExists := bearer != "" && loader.ExistsInModelPath(bearer)
-
-// 	// If no model was specified, take the first available
-// 	if modelFile == "" && !bearerExists && randomModel {
-// 		models, _ := loader.ListModels()
-// 		if len(models) > 0 {
-// 			modelFile = models[0]
-// 			log.Debug().Msgf("No model specified, using: %s", modelFile)
-// 		} else {
-// 			log.Debug().Msgf("No model specified, returning error")
-// 			return "", nil, fmt.Errorf("no model specified")
-// 		}
-// 	}
-
-// 	// If a model is found in bearer token takes precedence
-// 	if bearerExists {
-// 		log.Debug().Msgf("Using model from bearer token: %s", bearer)
-// 		modelFile = bearer
-// 	}
-// 	return modelFile, input, nil
-// }
-
-// func readConfig(modelFile string, input *OpenAIRequest, cm *ConfigMerger, loader *model.ModelLoader, debug bool, threads, ctx int, f16 bool) (*Config, *OpenAIRequest, error) {
-// 	// Load a config file if present after the model name
-// 	modelConfig := filepath.Join(loader.ModelPath, modelFile+".yaml")
-// 	if _, err := os.Stat(modelConfig); err == nil {
-// 		if err := cm.LoadConfig(modelConfig); err != nil {
-// 			return nil, nil, fmt.Errorf("failed loading model config (%s) %s", modelConfig, err.Error())
-// 		}
-// 	}
-
-// 	var config *Config
-// 	cfg, exists := cm.GetConfig(modelFile)
-// 	if !exists {
-// 		config = &Config{
-// 			OpenAIRequest: defaultRequest(modelFile),
-// 			ContextSize:   ctx,
-// 			Threads:       threads,
-// 			F16:           f16,
-// 			Debug:         debug,
-// 		}
-// 	} else {
-// 		config = &cfg
-// 	}
-
-// 	// Set the parameters for the language model prediction
-// 	updateConfig(config, input)
-
-// 	// Don't allow 0 as setting
-// 	if config.Threads == 0 {
-// 		if threads != 0 {
-// 			config.Threads = threads
-// 		} else {
-// 			config.Threads = 4
-// 		}
-// 	}
-
-// 	// Enforce debug flag if passed from CLI
-// 	if debug {
-// 		config.Debug = true
-// 	}
-
-// 	return config, input, nil
-// }

apiv2/localai.go

@@ -64,7 +64,7 @@ func combineRequestAndConfig[RequestType any](configManager *ConfigManager, mode
 }
 
 func (las *LocalAIServer) loadModel(configStub ConfigStub) {
-
+	
 }
 
 // CancelFineTune implements StrictServerInterface