mirror of
https://github.com/mudler/LocalAI.git
synced 2025-06-17 16:25:00 +00:00
feat: Add backend gallery (#5607)
* feat: Add backend gallery This PR add support to manage backends as similar to models. There is now available a backend gallery which can be used to install and remove extra backends. The backend gallery can be configured similarly as a model gallery, and API calls allows to install and remove new backends in runtime, and as well during the startup phase of LocalAI. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add backends docs Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * wip: Backend Dockerfile for python backends Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat: drop extras images, build python backends separately Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixup on all backends Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * test CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Tweaks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Drop old backends leftovers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixup CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Move dockerfile upper Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fix proto Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Feature dropped for consistency - we prefer model galleries Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add missing packages in the build image Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * exllama is ponly available on cublas Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * pin torch on chatterbox Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixups to index Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Debug CI * Install accellerators deps Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add target arch * Add cuda minor version Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Use self-hosted runners Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: use quay for test images Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixups for vllm and chatterbox Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Small fixups on CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chatterbox is only available for nvidia Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Simplify CI builds Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Adapt test, use qwen3 Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chore(model gallery): add jina-reranker-v1-tiny-en-gguf Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(gguf-parser): recover from potential panics that can happen while reading ggufs with gguf-parser Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Use reranker from llama.cpp in AIO images Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Limit concurrent jobs Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>
This commit is contained in:
Ettore Di Giacinto
GitHub
parent
a7a6020328
commit
2d64269763
114 changed files with 3996 additions and 1382 deletions
151
core/gallery/backends_test.go
Normal file
151
core/gallery/backends_test.go
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
package gallery
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("Gallery Backends", func() {
|
||||
var (
|
||||
tempDir string
|
||||
galleries []config.Gallery
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
var err error
|
||||
tempDir, err = os.MkdirTemp("", "gallery-test-*")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Setup test galleries
|
||||
galleries = []config.Gallery{
|
||||
{
|
||||
Name: "test-gallery",
|
||||
URL: "https://gist.githubusercontent.com/mudler/71d5376bc2aa168873fa519fa9f4bd56/raw/0557f9c640c159fa8e4eab29e8d98df6a3d6e80f/backend-gallery.yaml",
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
os.RemoveAll(tempDir)
|
||||
})
|
||||
|
||||
Describe("InstallBackendFromGallery", func() {
|
||||
It("should return error when backend is not found", func() {
|
||||
err := InstallBackendFromGallery(galleries, "non-existent", tempDir, nil)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("no model found with name"))
|
||||
})
|
||||
|
||||
It("should install backend from gallery", func() {
|
||||
err := InstallBackendFromGallery(galleries, "test-backend", tempDir, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(filepath.Join(tempDir, "test-backend", "run.sh")).To(BeARegularFile())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("InstallBackend", func() {
|
||||
It("should create base path if it doesn't exist", func() {
|
||||
newPath := filepath.Join(tempDir, "new-path")
|
||||
backend := GalleryBackend{
|
||||
Metadata: Metadata{
|
||||
Name: "test-backend",
|
||||
},
|
||||
URI: "test-uri",
|
||||
}
|
||||
|
||||
err := InstallBackend(newPath, &backend, nil)
|
||||
Expect(err).To(HaveOccurred()) // Will fail due to invalid URI, but path should be created
|
||||
Expect(newPath).To(BeADirectory())
|
||||
})
|
||||
|
||||
It("should create alias file when specified", func() {
|
||||
backend := GalleryBackend{
|
||||
Metadata: Metadata{
|
||||
Name: "test-backend",
|
||||
},
|
||||
URI: "quay.io/mudler/tests:localai-backend-test",
|
||||
Alias: "test-alias",
|
||||
}
|
||||
|
||||
err := InstallBackend(tempDir, &backend, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(filepath.Join(tempDir, "test-backend", "alias")).To(BeARegularFile())
|
||||
content, err := os.ReadFile(filepath.Join(tempDir, "test-backend", "alias"))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(string(content)).To(ContainSubstring("test-alias"))
|
||||
Expect(filepath.Join(tempDir, "test-backend", "run.sh")).To(BeARegularFile())
|
||||
|
||||
// Check that the alias was recognized
|
||||
backends, err := ListSystemBackends(tempDir)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(backends).To(HaveKeyWithValue("test-alias", filepath.Join(tempDir, "test-backend", "run.sh")))
|
||||
Expect(backends).To(HaveKeyWithValue("test-backend", filepath.Join(tempDir, "test-backend", "run.sh")))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("DeleteBackendFromSystem", func() {
|
||||
It("should delete backend directory", func() {
|
||||
backendName := "test-backend"
|
||||
backendPath := filepath.Join(tempDir, backendName)
|
||||
|
||||
// Create a dummy backend directory
|
||||
err := os.MkdirAll(backendPath, 0750)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = DeleteBackendFromSystem(tempDir, backendName)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(backendPath).NotTo(BeADirectory())
|
||||
})
|
||||
|
||||
It("should not error when backend doesn't exist", func() {
|
||||
err := DeleteBackendFromSystem(tempDir, "non-existent")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("ListSystemBackends", func() {
|
||||
It("should list backends without aliases", func() {
|
||||
// Create some dummy backend directories
|
||||
backendNames := []string{"backend1", "backend2", "backend3"}
|
||||
for _, name := range backendNames {
|
||||
err := os.MkdirAll(filepath.Join(tempDir, name), 0750)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
backends, err := ListSystemBackends(tempDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(backends).To(HaveLen(len(backendNames)))
|
||||
|
||||
for _, name := range backendNames {
|
||||
Expect(backends).To(HaveKeyWithValue(name, filepath.Join(tempDir, name, "run.sh")))
|
||||
}
|
||||
})
|
||||
|
||||
It("should handle backends with aliases", func() {
|
||||
backendName := "backend1"
|
||||
alias := "alias1"
|
||||
backendPath := filepath.Join(tempDir, backendName)
|
||||
|
||||
// Create backend directory
|
||||
err := os.MkdirAll(backendPath, 0750)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Create alias file
|
||||
err = os.WriteFile(filepath.Join(backendPath, "alias"), []byte(alias), 0644)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
backends, err := ListSystemBackends(tempDir)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(backends).To(HaveKeyWithValue(alias, filepath.Join(tempDir, backendName, "run.sh")))
|
||||
})
|
||||
|
||||
It("should return error when base path doesn't exist", func() {
|
||||
_, err := ListSystemBackends(filepath.Join(tempDir, "non-existent"))
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue