dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-30 06:30:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

feat: add PromptCacheRO option

Browse source
This commit is contained in:
mudler 2023-06-07 14:51:23 +02:00
parent 3b5df11881
commit 95bf290e37
5 changed files with 13 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

5
api/prediction.go
View file

@ -45,6 +45,7 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
if c.Batch != 0 {
llamaOpts = append(llamaOpts, llama.SetNBatch(c.Batch))
}
return llamaOpts
}
@ -174,6 +175,10 @@ func buildLLamaPredictOptions(c Config, modelPath string) []llama.PredictOption
predictOptions = append(predictOptions, llama.EnablePromptCacheAll)
}
if c.PromptCacheRO {
predictOptions = append(predictOptions, llama.EnablePromptCacheRO)
}
if c.PromptCachePath != "" {
// Create parent directory
p := filepath.Join(modelPath, c.PromptCachePath)