dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-30 06:30:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

feat: add experimental support for falcon-7b

Browse source 
This is from the ggml example:
https://github.com/ggerganov/ggml/pull/231/files.

go-ggml-transformers commit:
bd765bb6f3

Signed-off-by: mudler <mudler@mocaccino.org>
This commit is contained in:
mudler 2023-06-06 15:24:26 +02:00
parent 49a2b30350
commit a06a8a96c8
4 changed files with 36 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

24
api/prediction.go
View file

@ -368,6 +368,30 @@ func ModelInference(s string, loader *model.ModelLoader, c Config, tokenCallback
predictOptions = append(predictOptions, bloomz.SetSeed(c.Seed))
}
return model.Predict(
s,
predictOptions...,
)
}
case *transformers.Falcon:
fn = func() (string, error) {
// Generate the prediction using the language model
predictOptions := []transformers.PredictOption{
transformers.SetTemperature(c.Temperature),
transformers.SetTopP(c.TopP),
transformers.SetTopK(c.TopK),
transformers.SetTokens(c.Maxtokens),
transformers.SetThreads(c.Threads),
}
if c.Batch != 0 {
predictOptions = append(predictOptions, transformers.SetBatch(c.Batch))
}
if c.Seed != 0 {
predictOptions = append(predictOptions, transformers.SetSeed(c.Seed))
}
return model.Predict(
s,
predictOptions...,