From e28e80857b1aebbb8bb57c88664e9cdcddb60150 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 5 Oct 2024 10:41:35 +0200 Subject: [PATCH] feat(shutdown): allow force shutdown of backends (#3733) We default to a soft kill, however, we might want to force killing backends after a while to avoid hanging requests (which may hallucinate indefinetly) Signed-off-by: Ettore Di Giacinto --- pkg/model/loader.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/model/loader.go b/pkg/model/loader.go index 0dcc9915..52f479c5 100644 --- a/pkg/model/loader.go +++ b/pkg/model/loader.go @@ -157,6 +157,11 @@ func (ml *ModelLoader) ShutdownModel(modelName string) error { } time.Sleep(dur) retries++ + + if retries > 10 && os.Getenv("LOCALAI_FORCE_BACKEND_SHUTDOWN") == "true" { + log.Warn().Msgf("Model %s is still busy after %d retries. Forcing shutdown.", modelName, retries) + break + } } return ml.deleteProcess(modelName)