feat: Add backend gallery (#5607)

* feat: Add backend gallery

This PR add support to manage backends as similar to models. There is
now available a backend gallery which can be used to install and remove
extra backends.
The backend gallery can be configured similarly as a model gallery, and
API calls allows to install and remove new backends in runtime, and as
well during the startup phase of LocalAI.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add backends docs

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* wip: Backend Dockerfile for python backends

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat: drop extras images, build python backends separately

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fixup on all backends

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* test CI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Tweaks

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Drop old backends leftovers

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fixup CI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Move dockerfile upper

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fix proto

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Feature dropped for consistency - we prefer model galleries

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add missing packages in the build image

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* exllama is ponly available on cublas

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* pin torch on chatterbox

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fixups to index

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* CI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Debug CI

* Install accellerators deps

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add target arch

* Add cuda minor version

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Use self-hosted runners

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* ci: use quay for test images

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fixups for vllm and chatterbox

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Small fixups on CI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chatterbox is only available for nvidia

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Simplify CI builds

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Adapt test, use qwen3

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chore(model gallery): add jina-reranker-v1-tiny-en-gguf

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix(gguf-parser): recover from potential panics that can happen while reading ggufs with gguf-parser

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Use reranker from llama.cpp in AIO images

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Limit concurrent jobs

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>

core/gallery/backends.go

@@ -0,0 +1,107 @@
+package gallery
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/oci"
+)
+
+// Installs a model from the gallery
+func InstallBackendFromGallery(galleries []config.Gallery, name string, basePath string, downloadStatus func(string, string, string, float64)) error {
+	backends, err := AvailableBackends(galleries, basePath)
+	if err != nil {
+		return err
+	}
+
+	backend := FindGalleryElement(backends, name, basePath)
+	if backend == nil {
+		return fmt.Errorf("no model found with name %q", name)
+	}
+
+	return InstallBackend(basePath, backend, downloadStatus)
+}
+
+func InstallBackend(basePath string, config *GalleryBackend, downloadStatus func(string, string, string, float64)) error {
+	// Create base path if it doesn't exist
+	err := os.MkdirAll(basePath, 0750)
+	if err != nil {
+		return fmt.Errorf("failed to create base path: %v", err)
+	}
+
+	name := config.Name
+
+	img, err := oci.GetImage(config.URI, "", nil, nil)
+	if err != nil {
+		return fmt.Errorf("failed to get image %q: %v", config.URI, err)
+	}
+
+	backendPath := filepath.Join(basePath, name)
+	if err := os.MkdirAll(backendPath, 0750); err != nil {
+		return fmt.Errorf("failed to create backend path %q: %v", backendPath, err)
+	}
+
+	if err := oci.ExtractOCIImage(img, backendPath); err != nil {
+		return fmt.Errorf("failed to extract image %q: %v", config.URI, err)
+	}
+
+	if config.Alias != "" {
+		// Write an alias file inside
+		aliasFile := filepath.Join(backendPath, "alias")
+		if err := os.WriteFile(aliasFile, []byte(config.Alias), 0644); err != nil {
+			return fmt.Errorf("failed to write alias file %q: %v", aliasFile, err)
+		}
+	}
+
+	return nil
+}
+
+func DeleteBackendFromSystem(basePath string, name string) error {
+	backendFile := filepath.Join(basePath, name)
+
+	return os.RemoveAll(backendFile)
+}
+
+func ListSystemBackends(basePath string) (map[string]string, error) {
+	backends, err := os.ReadDir(basePath)
+	if err != nil {
+		return nil, err
+	}
+
+	backendsNames := make(map[string]string)
+
+	for _, backend := range backends {
+		if backend.IsDir() {
+			runFile := filepath.Join(basePath, backend.Name(), "run.sh")
+			backendsNames[backend.Name()] = runFile
+
+			aliasFile := filepath.Join(basePath, backend.Name(), "alias")
+			if _, err := os.Stat(aliasFile); err == nil {
+				// read the alias file, and use it as key
+				alias, err := os.ReadFile(aliasFile)
+				if err != nil {
+					return nil, err
+				}
+				backendsNames[string(alias)] = runFile
+			}
+		}
+	}
+
+	return backendsNames, nil
+}
+
+func RegisterBackends(basePath string, modelLoader *model.ModelLoader) error {
+	backends, err := ListSystemBackends(basePath)
+	if err != nil {
+		return err
+	}
+
+	for name, runFile := range backends {
+		modelLoader.SetExternalBackend(name, runFile)
+	}
+
+	return nil
+}