dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-28 06:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 12

feat(llama.cpp): estimate vram usage (#5299)

Browse source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto 2025-05-02 17:40:26 +02:00 • committed by GitHub
parent bace6516f1
commit 5c6cd50ed6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 131 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

9
core/config/guesser.go
View file

@ -4,9 +4,8 @@ import (
"os"
"path/filepath"
"github.com/mudler/LocalAI/pkg/xsysinfo"
gguf "github.com/gpustack/gguf-parser-go"
"github.com/rs/zerolog/log"
gguf "github.com/thxcode/gguf-parser-go"
)
func guessDefaultsFromFile(cfg *BackendConfig, modelPath string, defaultCtx int) {
@ -36,10 +35,4 @@ func guessDefaultsFromFile(cfg *BackendConfig, modelPath string, defaultCtx int)
}
cfg.ContextSize = &defaultCtx
}
if cfg.Options == nil {
if xsysinfo.HasGPU("nvidia") || xsysinfo.HasGPU("amd") {
cfg.Options = []string{"gpu"}
}
}
}