dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-22 11:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

wip

Browse source
This commit is contained in:
Ettore Di Giacinto 2023-11-08 20:05:54 +01:00
parent f0e265a96d
commit 78ef045bb3
21 changed files with 485 additions and 336 deletions
Download patch file Download diff file Expand all files Collapse all files

3
api/backend/llm.go
View file

@ -26,7 +26,7 @@ type TokenUsage struct {
Completion int
}
func ModelInference(ctx context.Context, s string, loader *model.ModelLoader, c config.Config, o *options.Option, tokenCallback func(string, TokenUsage) bool) (func() (LLMResponse, error), error) {
func ModelInference(ctx context.Context, s string, images []string, loader *model.ModelLoader, c config.Config, o *options.Option, tokenCallback func(string, TokenUsage) bool) (func() (LLMResponse, error), error) {
modelFile := c.Model
grpcOpts := gRPCModelOpts(c)
@ -72,6 +72,7 @@ func ModelInference(ctx context.Context, s string, loader *model.ModelLoader, c
fn := func() (LLMResponse, error) {
opts := gRPCPredictOpts(c, loader.ModelPath)
opts.Prompt = s
opts.Images = images
tokenUsage := TokenUsage{}

1
api/backend/options.go
View file

@ -45,6 +45,7 @@ func gRPCModelOpts(c config.Config) *pb.ModelOptions {
DraftModel: c.DraftModel,
AudioPath: c.VallE.AudioPath,
Quantization: c.Quantization,
MMProj: c.MMProj,
LoraAdapter: c.LoraAdapter,
LoraBase: c.LoraBase,
NGQA: c.NGQA,