dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-20 02:24:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 21

Enhancements (#34)

Browse source 
Signed-off-by: mudler <mudler@c3os.io>
This commit is contained in:
Ettore Di Giacinto 2023-04-19 17:10:29 +02:00 • committed by GitHub
parent a9a875ee2b
commit 7fec26f5d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 226 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

14
main.go
View file

@ -34,11 +34,6 @@ var nonEmptyInput string = `Below is an instruction that describes a task, paire
### Response:
`
func llamaFromOptions(ctx *cli.Context) (*llama.LLama, error) {
opts := []llama.ModelOption{llama.SetContext(ctx.Int("context-size"))}
return llama.New(ctx.String("model"), opts...)
}
func templateString(t string, in interface{}) (string, error) {
// Parse the template
tmpl, err := template.New("prompt").Parse(t)
@ -125,6 +120,10 @@ echo "An Alpaca (Vicugna pacos) is a domesticated species of South American came
Name: "api",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "f16",
EnvVars: []string{"F16"},
},
&cli.IntFlag{
Name: "threads",
EnvVars: []string{"THREADS"},
@ -146,7 +145,7 @@ echo "An Alpaca (Vicugna pacos) is a domesticated species of South American came
},
},
Action: func(ctx *cli.Context) error {
return api.Start(model.NewModelLoader(ctx.String("models-path")), ctx.String("address"), ctx.Int("threads"))
return api.Start(model.NewModelLoader(ctx.String("models-path")), ctx.String("address"), ctx.Int("threads"), ctx.Int("context-size"), ctx.Bool("f16"))
},
},
},
@ -201,7 +200,8 @@ echo "An Alpaca (Vicugna pacos) is a domesticated species of South American came
os.Exit(1)
}
l, err := llamaFromOptions(ctx)
opts := []llama.ModelOption{llama.SetContext(ctx.Int("context-size"))}
l, err := llama.New(ctx.String("model"), opts...)
if err != nil {
fmt.Println("Loading the model failed:", err.Error())
os.Exit(1)