From 6636badda7cd8412b7e7b3099ba6afce6acb68fd Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 3 May 2025 10:35:06 +0200 Subject: [PATCH] chore(defaults): enlarge defaults, drop gpu layers which is infered Signed-off-by: Ettore Di Giacinto --- core/backend/options.go | 2 +- core/config/backend_config.go | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/core/backend/options.go b/core/backend/options.go index 56cf3385..ab602b1d 100644 --- a/core/backend/options.go +++ b/core/backend/options.go @@ -99,7 +99,7 @@ func grpcModelOpts(c config.BackendConfig) *pb.ModelOptions { mmap = *c.MMap } - ctxSize := 1024 + ctxSize := 4096 if c.ContextSize != nil { ctxSize = *c.ContextSize } diff --git a/core/config/backend_config.go b/core/config/backend_config.go index cb1263a6..5c436400 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -304,9 +304,6 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) { defaultTFZ := 1.0 defaultZero := 0 - // Try to offload all GPU layers (if GPU is found) - defaultHigh := 99999999 - trueV := true falseV := false @@ -366,9 +363,6 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) { if cfg.MirostatTAU == nil { cfg.MirostatTAU = &defaultMirostatTAU } - if cfg.NGPULayers == nil { - cfg.NGPULayers = &defaultHigh - } if cfg.LowVRAM == nil { cfg.LowVRAM = &falseV