dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-30 06:30:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

swap devbox again, broken momentarily due to codegen, investigate multiple allOf hierarchy

Browse source
This commit is contained in:
Dave Lee 2023-06-09 01:38:11 -04:00
parent c24044d6a1
commit 0ee645e091
8 changed files with 108 additions and 161 deletions
Download patch file Download diff file Expand all files Collapse all files

13
apiv2/config.go
View file

@ -168,12 +168,15 @@ func (sc SpecificConfig[RequestModel]) ToPredictOptions() []llama.PredictOption
llamaOpts = append(llamaOpts, llama.SetPenalty(float64(*req.FrequencyPenalty))) // CAST
}
if stop0, err := req.Stop.AsCreateChatCompletionRequestStop0(); err == nil {
llamaOpts = append(llamaOpts, llama.SetStopWords(stop0))
}
if req.Stop != nil {
if stop1, err := req.Stop.AsCreateChatCompletionRequestStop1(); err == nil && len(stop1) > 0 {
llamaOpts = append(llamaOpts, llama.SetStopWords(stop1...))
if stop0, err := req.Stop.AsCreateChatCompletionRequestStop0(); err == nil {
llamaOpts = append(llamaOpts, llama.SetStopWords(stop0))
}
if stop1, err := req.Stop.AsCreateChatCompletionRequestStop1(); err == nil && len(stop1) > 0 {
llamaOpts = append(llamaOpts, llama.SetStopWords(stop1...))
}
}
if req.XLocalaiExtensions != nil {

14
apiv2/engine.go
View file

@ -93,6 +93,7 @@ func (e *LocalAIEngine) GetModelPredictionFunction(config Config, tokenCallback
localModel.SetTokenCallback(tokenCallback)
}
tempFakePO := []gpt4all.PredictOption{}
mappedPredictOptions := gpt4all.PredictOptions{}
mapstructure.Decode(config.ToPredictOptions(), &mappedPredictOptions)
@ -101,7 +102,7 @@ func (e *LocalAIEngine) GetModelPredictionFunction(config Config, tokenCallback
str, err := localModel.Predict(
p.AsString(),
// mappedPredictOptions,
nil,
tempFakePO...,
)
// Seems that if we don't free the callback explicitly we leave functions registered (that might try to send on closed channels)
// For instance otherwise the API returns: {"error":{"code":500,"message":"send on closed channel","type":""}}
@ -163,6 +164,13 @@ func (e *LocalAIEngine) GetModelPredictionFunction(config Config, tokenCallback
for n_i := 0; n_i < n; n_i++ {
res, err := predictOnce(prompt)
if err != nil {
fmt.Printf("ERROR DURING GetModelPredictionFunction -> PredictionFunction for %T with p_i: %d/n_i: %d\n%s", config, p_i, n_i, err.Error())
return nil, err
}
fmt.Printf("\n\n🤯 raw res: %s\n\n", res)
// TODO: this used to be a part of finetune. For.... questionable parameter reasons I've moved it up here. Revisit this if it's smelly in the future.
ccr, is_ccr := req.(CreateCompletionRequest)
if is_ccr {
@ -173,10 +181,6 @@ func (e *LocalAIEngine) GetModelPredictionFunction(config Config, tokenCallback
res = e.Finetune(config, res)
if err != nil {
fmt.Printf("ERROR DURING GetModelPredictionFunction -> PredictionFunction for %T with p_i: %d/n_i: %d\n%s", config, p_i, n_i, err.Error())
return nil, err
}
if tokenCallback != nil && !supportStreams {
tokenCallback(res)
}

1
apiv2/localai.go
View file

@ -102,6 +102,7 @@ func (las *LocalAIServer) CreateChatCompletion(ctx context.Context, request Crea
return nil, err
}
fmt.Println("About to call predict()")
predictions, err := predict()
if err != nil {
fmt.Printf("!!!!!!!!!! Error INSIDE predict fn %s\n", err.Error())