From 77ff2f9c4daaa644b8415207a5306415f2e4fa9a Mon Sep 17 00:00:00 2001 From: mudler Date: Wed, 14 Jun 2023 18:22:07 +0200 Subject: [PATCH] fix: set default batch size --- api/prediction.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/prediction.go b/api/prediction.go index ff4866f0..7a49a57e 100644 --- a/api/prediction.go +++ b/api/prediction.go @@ -44,6 +44,8 @@ func defaultLLamaOpts(c Config) []llama.ModelOption { llamaOpts = append(llamaOpts, llama.SetTensorSplit(c.TensorSplit)) if c.Batch != 0 { llamaOpts = append(llamaOpts, llama.SetNBatch(c.Batch)) + } else { + llamaOpts = append(llamaOpts, llama.SetNBatch(512)) } return llamaOpts