dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-20 10:35:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 21

feat(grpc): backend SPI pluggable in embedding mode (#1621)

Browse source 
* run server

* grpc backend embedded support

* backend providable
This commit is contained in:
coyzeng 2024-01-23 15:56:36 +08:00 • committed by GitHub
parent efe2883c5d
commit d5d82ba344
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 196 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

2
api/backend/embeddings.go
View file

@ -41,7 +41,7 @@ func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, c config.
var fn func() ([]float32, error)
switch model := inferenceModel.(type) {
case *grpc.Client:
case grpc.Backend:
fn = func() ([]float32, error) {
predictOptions := gRPCPredictOpts(c, loader.ModelPath)
if len(tokens) > 0 {

2
api/backend/llm.go
View file

@ -31,7 +31,7 @@ func ModelInference(ctx context.Context, s string, images []string, loader *mode
grpcOpts := gRPCModelOpts(c)
var inferenceModel *grpc.Client
var inferenceModel grpc.Backend
var err error
opts := modelOpts(c, o, []model.Option{