From 506905ae9b4fdff1223cef4da44a8079d78cd78e Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Tue, 6 Jun 2023 12:21:16 -0400 Subject: [PATCH] switching workstations, stash the merge --- apiv2/config.go | 228 ++++------------------------------------------- apiv2/localai.go | 2 +- 2 files changed, 20 insertions(+), 210 deletions(-) diff --git a/apiv2/config.go b/apiv2/config.go index ff6aa026..d083bd03 100644 --- a/apiv2/config.go +++ b/apiv2/config.go @@ -31,12 +31,6 @@ type SpecificConfig[RequestModel any] struct { RequestDefaults RequestModel `yaml:"request_defaults" mapstructure:"request_defaults"` } -// type Config struct { -// Registration ConfigRegistration `yaml:"registration"` -// LocalPaths ConfigLocalPaths `yaml:"local_paths"` -// RequestDefaults interface{} `yaml:"request_defaults"` -// } - type Config interface { GetRequestDefaults() interface{} GetLocalPaths() ConfigLocalPaths @@ -198,212 +192,28 @@ func (cm *ConfigManager) ListConfigs() []ConfigRegistration { return cm.listConfigs() } -// // Not sure about this one, but it seems like a decent place to stick it for an experiment at least. -// func (cm *ConfigManager) GetTextConfigForRequest() +// These functions I'm a bit dubious about. I think there's a better refactoring down in pkg/model +// But to get a minimal test up and running, here we go! -// func (cm *ConfigMerger) LoadConfigs(path string) error { -// cm.Lock() -// defer cm.Unlock() -// files, err := ioutil.ReadDir(path) -// if err != nil { -// return err -// } +func (sc SpecificConfig[RequestModel]) ToModelOptions() []llama.ModelOption { -// for _, file := range files { -// // Skip templates, YAML and .keep files -// if !strings.Contains(file.Name(), ".yaml") { -// continue -// } -// c, err := ReadConfig(filepath.Join(path, file.Name())) -// if err == nil { -// cm.configs[ConfigLookup{Name: c.Name, Endpoint: c.Endpoint}] = *c -// } -// } + llamaOpts := []llama.ModelOption{} -// return nil -// } + // Code to Port: -// func (cm *ConfigMerger) Get + // if c.ContextSize != 0 { + // llamaOpts = append(llamaOpts, llama.SetContext(c.ContextSize)) + // } + // if c.F16 { + // llamaOpts = append(llamaOpts, llama.EnableF16Memory) + // } + // if c.Embeddings { + // llamaOpts = append(llamaOpts, llama.EnableEmbeddings) + // } -// func updateConfig(config *Config, input *OpenAIRequest) { -// if input.Echo { -// config.Echo = input.Echo -// } -// if input.TopK != 0 { -// config.TopK = input.TopK -// } -// if input.TopP != 0 { -// config.TopP = input.TopP -// } + // if c.NGPULayers != 0 { + // llamaOpts = append(llamaOpts, llama.SetGPULayers(c.NGPULayers)) + // } -// if input.Temperature != 0 { -// config.Temperature = input.Temperature -// } - -// if input.Maxtokens != 0 { -// config.Maxtokens = input.Maxtokens -// } - -// switch stop := input.Stop.(type) { -// case string: -// if stop != "" { -// config.StopWords = append(config.StopWords, stop) -// } -// case []interface{}: -// for _, pp := range stop { -// if s, ok := pp.(string); ok { -// config.StopWords = append(config.StopWords, s) -// } -// } -// } - -// if input.RepeatPenalty != 0 { -// config.RepeatPenalty = input.RepeatPenalty -// } - -// if input.Keep != 0 { -// config.Keep = input.Keep -// } - -// if input.Batch != 0 { -// config.Batch = input.Batch -// } - -// if input.F16 { -// config.F16 = input.F16 -// } - -// if input.IgnoreEOS { -// config.IgnoreEOS = input.IgnoreEOS -// } - -// if input.Seed != 0 { -// config.Seed = input.Seed -// } - -// if input.Mirostat != 0 { -// config.Mirostat = input.Mirostat -// } - -// if input.MirostatETA != 0 { -// config.MirostatETA = input.MirostatETA -// } - -// if input.MirostatTAU != 0 { -// config.MirostatTAU = input.MirostatTAU -// } - -// switch inputs := input.Input.(type) { -// case string: -// if inputs != "" { -// config.InputStrings = append(config.InputStrings, inputs) -// } -// case []interface{}: -// for _, pp := range inputs { -// switch i := pp.(type) { -// case string: -// config.InputStrings = append(config.InputStrings, i) -// case []interface{}: -// tokens := []int{} -// for _, ii := range i { -// tokens = append(tokens, int(ii.(float64))) -// } -// config.InputToken = append(config.InputToken, tokens) -// } -// } -// } - -// switch p := input.Prompt.(type) { -// case string: -// config.PromptStrings = append(config.PromptStrings, p) -// case []interface{}: -// for _, pp := range p { -// if s, ok := pp.(string); ok { -// config.PromptStrings = append(config.PromptStrings, s) -// } -// } -// } -// } -// func readInput(c *fiber.Ctx, loader *model.ModelLoader, randomModel bool) (string, *OpenAIRequest, error) { -// input := new(OpenAIRequest) -// // Get input data from the request body -// if err := c.BodyParser(input); err != nil { -// return "", nil, err -// } - -// modelFile := input.Model - -// if c.Params("model") != "" { -// modelFile = c.Params("model") -// } - -// received, _ := json.Marshal(input) - -// log.Debug().Msgf("Request received: %s", string(received)) - -// // Set model from bearer token, if available -// bearer := strings.TrimLeft(c.Get("authorization"), "Bearer ") -// bearerExists := bearer != "" && loader.ExistsInModelPath(bearer) - -// // If no model was specified, take the first available -// if modelFile == "" && !bearerExists && randomModel { -// models, _ := loader.ListModels() -// if len(models) > 0 { -// modelFile = models[0] -// log.Debug().Msgf("No model specified, using: %s", modelFile) -// } else { -// log.Debug().Msgf("No model specified, returning error") -// return "", nil, fmt.Errorf("no model specified") -// } -// } - -// // If a model is found in bearer token takes precedence -// if bearerExists { -// log.Debug().Msgf("Using model from bearer token: %s", bearer) -// modelFile = bearer -// } -// return modelFile, input, nil -// } - -// func readConfig(modelFile string, input *OpenAIRequest, cm *ConfigMerger, loader *model.ModelLoader, debug bool, threads, ctx int, f16 bool) (*Config, *OpenAIRequest, error) { -// // Load a config file if present after the model name -// modelConfig := filepath.Join(loader.ModelPath, modelFile+".yaml") -// if _, err := os.Stat(modelConfig); err == nil { -// if err := cm.LoadConfig(modelConfig); err != nil { -// return nil, nil, fmt.Errorf("failed loading model config (%s) %s", modelConfig, err.Error()) -// } -// } - -// var config *Config -// cfg, exists := cm.GetConfig(modelFile) -// if !exists { -// config = &Config{ -// OpenAIRequest: defaultRequest(modelFile), -// ContextSize: ctx, -// Threads: threads, -// F16: f16, -// Debug: debug, -// } -// } else { -// config = &cfg -// } - -// // Set the parameters for the language model prediction -// updateConfig(config, input) - -// // Don't allow 0 as setting -// if config.Threads == 0 { -// if threads != 0 { -// config.Threads = threads -// } else { -// config.Threads = 4 -// } -// } - -// // Enforce debug flag if passed from CLI -// if debug { -// config.Debug = true -// } - -// return config, input, nil -// } + return llamaOpts +} \ No newline at end of file diff --git a/apiv2/localai.go b/apiv2/localai.go index 4bc945a8..3627f6b6 100644 --- a/apiv2/localai.go +++ b/apiv2/localai.go @@ -64,7 +64,7 @@ func combineRequestAndConfig[RequestType any](configManager *ConfigManager, mode } func (las *LocalAIServer) loadModel(configStub ConfigStub) { - + } // CancelFineTune implements StrictServerInterface