dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-28 05:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 22

feat: tokenization

Browse source 
Signed-off-by: shraddhazpy <shraddha@shraddhafive.in>
This commit is contained in:
shraddhazpy 2025-01-31 09:28:46 +05:30
parent 1d6afbd65d
commit 14df08c6e1
3 changed files with 19 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

11
core/backend/tokenize.go
View file

@ -16,12 +16,7 @@ func ModelTokenize(s string, loader *model.ModelLoader, backendConfig config.Bac
opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
if backendConfig.Backend == "" {
inferenceModel, err = loader.Load(opts...)
} else {
opts = append(opts, model.WithBackendString(backendConfig.Backend))
inferenceModel, err = loader.Load(opts...)
}
inferenceModel, err = loader.Load(opts...)
if err != nil {
return schema.TokenizeResponse{}, err
}
@ -35,6 +30,10 @@ func ModelTokenize(s string, loader *model.ModelLoader, backendConfig config.Bac
return schema.TokenizeResponse{}, err
}
if resp.Tokens == nil {
resp.Tokens = make([]int32, 0)
}
return schema.TokenizeResponse{
Tokens: resp.Tokens,
}, nil