dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-22 19:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

feat: update llama, enable NUMA (#684)

Browse source
This commit is contained in:
Ettore Di Giacinto 2023-06-27 09:00:10 +02:00 • committed by GitHub
parent e130b208ab
commit 3593cb0c87
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 6 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
api/prediction.go
View file

@ -48,6 +48,10 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
llamaOpts = append(llamaOpts, llama.SetNBatch(512))
}
if c.NUMA {
llamaOpts = append(llamaOpts, llama.EnableNUMA)
}
if c.LowVRAM {
llamaOpts = append(llamaOpts, llama.EnabelLowVRAM)
}