dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-20 18:45:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 21

chore(defaults): enlarge defaults, drop gpu layers which is infered

Browse source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto 2025-05-03 10:35:06 +02:00
parent 1da0644aa3
commit 6636badda7
2 changed files with 1 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
core/backend/options.go
View file

@ -99,7 +99,7 @@ func grpcModelOpts(c config.BackendConfig) *pb.ModelOptions {
mmap = *c.MMap mmap = *c.MMap
} }
ctxSize := 1024 ctxSize := 4096
if c.ContextSize != nil { if c.ContextSize != nil {
ctxSize = *c.ContextSize ctxSize = *c.ContextSize
} }

6
core/config/backend_config.go
View file

@ -304,9 +304,6 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
defaultTFZ := 1.0 defaultTFZ := 1.0
defaultZero := 0 defaultZero := 0
// Try to offload all GPU layers (if GPU is found)
defaultHigh := 99999999
trueV := true trueV := true
falseV := false falseV := false
@ -366,9 +363,6 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
if cfg.MirostatTAU == nil { if cfg.MirostatTAU == nil {
cfg.MirostatTAU = &defaultMirostatTAU cfg.MirostatTAU = &defaultMirostatTAU
} }
if cfg.NGPULayers == nil {
cfg.NGPULayers = &defaultHigh
}
if cfg.LowVRAM == nil { if cfg.LowVRAM == nil {
cfg.LowVRAM = &falseV cfg.LowVRAM = &falseV