dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-17 08:15:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 23

feat: Add backend gallery (#5607)

Browse source 
* feat: Add backend gallery

This PR add support to manage backends as similar to models. There is
now available a backend gallery which can be used to install and remove
extra backends.
The backend gallery can be configured similarly as a model gallery, and
API calls allows to install and remove new backends in runtime, and as
well during the startup phase of LocalAI.

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add backends docs

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* wip: Backend Dockerfile for python backends

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat: drop extras images, build python backends separately

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fixup on all backends

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* test CI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Tweaks

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Drop old backends leftovers

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fixup CI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Move dockerfile upper

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fix proto

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Feature dropped for consistency - we prefer model galleries

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add missing packages in the build image

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* exllama is ponly available on cublas

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* pin torch on chatterbox

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fixups to index

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* CI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Debug CI

* Install accellerators deps

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add target arch

* Add cuda minor version

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Use self-hosted runners

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* ci: use quay for test images

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fixups for vllm and chatterbox

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Small fixups on CI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chatterbox is only available for nvidia

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Simplify CI builds

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Adapt test, use qwen3

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chore(model gallery): add jina-reranker-v1-tiny-en-gguf

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix(gguf-parser): recover from potential panics that can happen while reading ggufs with gguf-parser

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Use reranker from llama.cpp in AIO images

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Limit concurrent jobs

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>
This commit is contained in:
Ettore Di Giacinto 2025-06-15 14:56:52 +02:00 • committed by GitHub
parent a7a6020328
commit 2d64269763
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
114 changed files with 3996 additions and 1382 deletions
Download patch file Download diff file Expand all files Collapse all files

35
core/gallery/backend_types.go Normal file
View file

@ -0,0 +1,35 @@
package gallery
import "github.com/mudler/LocalAI/core/config"
type GalleryBackend struct {
Metadata `json:",inline" yaml:",inline"`
Alias string `json:"alias,omitempty" yaml:"alias,omitempty"`
URI string `json:"uri,omitempty" yaml:"uri,omitempty"`
}
type GalleryBackends []*GalleryBackend
func (m *GalleryBackend) SetGallery(gallery config.Gallery) {
m.Gallery = gallery
}
func (m *GalleryBackend) SetInstalled(installed bool) {
m.Installed = installed
}
func (m *GalleryBackend) GetName() string {
return m.Name
}
func (m *GalleryBackend) GetGallery() config.Gallery {
return m.Gallery
}
func (m *GalleryBackend) GetDescription() string {
return m.Description
}
func (m *GalleryBackend) GetTags() []string {
return m.Tags
}

107
core/gallery/backends.go Normal file
View file

@ -0,0 +1,107 @@
package gallery
import (
"fmt"
"os"
"path/filepath"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/oci"
)
// Installs a model from the gallery
func InstallBackendFromGallery(galleries []config.Gallery, name string, basePath string, downloadStatus func(string, string, string, float64)) error {
backends, err := AvailableBackends(galleries, basePath)
if err != nil {
return err
}
backend := FindGalleryElement(backends, name, basePath)
if backend == nil {
return fmt.Errorf("no model found with name %q", name)
}
return InstallBackend(basePath, backend, downloadStatus)
}
func InstallBackend(basePath string, config *GalleryBackend, downloadStatus func(string, string, string, float64)) error {
// Create base path if it doesn't exist
err := os.MkdirAll(basePath, 0750)
if err != nil {
return fmt.Errorf("failed to create base path: %v", err)
}
name := config.Name
img, err := oci.GetImage(config.URI, "", nil, nil)
if err != nil {
return fmt.Errorf("failed to get image %q: %v", config.URI, err)
}
backendPath := filepath.Join(basePath, name)
if err := os.MkdirAll(backendPath, 0750); err != nil {
return fmt.Errorf("failed to create backend path %q: %v", backendPath, err)
}
if err := oci.ExtractOCIImage(img, backendPath); err != nil {
return fmt.Errorf("failed to extract image %q: %v", config.URI, err)
}
if config.Alias != "" {
// Write an alias file inside
aliasFile := filepath.Join(backendPath, "alias")
if err := os.WriteFile(aliasFile, []byte(config.Alias), 0644); err != nil {
return fmt.Errorf("failed to write alias file %q: %v", aliasFile, err)
}
}
return nil
}
func DeleteBackendFromSystem(basePath string, name string) error {
backendFile := filepath.Join(basePath, name)
return os.RemoveAll(backendFile)
}
func ListSystemBackends(basePath string) (map[string]string, error) {
backends, err := os.ReadDir(basePath)
if err != nil {
return nil, err
}
backendsNames := make(map[string]string)
for _, backend := range backends {
if backend.IsDir() {
runFile := filepath.Join(basePath, backend.Name(), "run.sh")
backendsNames[backend.Name()] = runFile
aliasFile := filepath.Join(basePath, backend.Name(), "alias")
if _, err := os.Stat(aliasFile); err == nil {
// read the alias file, and use it as key
alias, err := os.ReadFile(aliasFile)
if err != nil {
return nil, err
}
backendsNames[string(alias)] = runFile
}
}
}
return backendsNames, nil
}
func RegisterBackends(basePath string, modelLoader *model.ModelLoader) error {
backends, err := ListSystemBackends(basePath)
if err != nil {
return err
}
for name, runFile := range backends {
modelLoader.SetExternalBackend(name, runFile)
}
return nil
}

151
core/gallery/backends_test.go Normal file
View file

@ -0,0 +1,151 @@
package gallery
import (
"os"
"path/filepath"
"github.com/mudler/LocalAI/core/config"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Gallery Backends", func() {
var (
tempDir string
galleries []config.Gallery
)
BeforeEach(func() {
var err error
tempDir, err = os.MkdirTemp("", "gallery-test-*")
Expect(err).NotTo(HaveOccurred())
// Setup test galleries
galleries = []config.Gallery{
{
Name: "test-gallery",
URL: "https://gist.githubusercontent.com/mudler/71d5376bc2aa168873fa519fa9f4bd56/raw/0557f9c640c159fa8e4eab29e8d98df6a3d6e80f/backend-gallery.yaml",
},
}
})
AfterEach(func() {
os.RemoveAll(tempDir)
})
Describe("InstallBackendFromGallery", func() {
It("should return error when backend is not found", func() {
err := InstallBackendFromGallery(galleries, "non-existent", tempDir, nil)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("no model found with name"))
})
It("should install backend from gallery", func() {
err := InstallBackendFromGallery(galleries, "test-backend", tempDir, nil)
Expect(err).ToNot(HaveOccurred())
Expect(filepath.Join(tempDir, "test-backend", "run.sh")).To(BeARegularFile())
})
})
Describe("InstallBackend", func() {
It("should create base path if it doesn't exist", func() {
newPath := filepath.Join(tempDir, "new-path")
backend := GalleryBackend{
Metadata: Metadata{
Name: "test-backend",
},
URI: "test-uri",
}
err := InstallBackend(newPath, &backend, nil)
Expect(err).To(HaveOccurred()) // Will fail due to invalid URI, but path should be created
Expect(newPath).To(BeADirectory())
})
It("should create alias file when specified", func() {
backend := GalleryBackend{
Metadata: Metadata{
Name: "test-backend",
},
URI: "quay.io/mudler/tests:localai-backend-test",
Alias: "test-alias",
}
err := InstallBackend(tempDir, &backend, nil)
Expect(err).ToNot(HaveOccurred())
Expect(filepath.Join(tempDir, "test-backend", "alias")).To(BeARegularFile())
content, err := os.ReadFile(filepath.Join(tempDir, "test-backend", "alias"))
Expect(err).ToNot(HaveOccurred())
Expect(string(content)).To(ContainSubstring("test-alias"))
Expect(filepath.Join(tempDir, "test-backend", "run.sh")).To(BeARegularFile())
// Check that the alias was recognized
backends, err := ListSystemBackends(tempDir)
Expect(err).ToNot(HaveOccurred())
Expect(backends).To(HaveKeyWithValue("test-alias", filepath.Join(tempDir, "test-backend", "run.sh")))
Expect(backends).To(HaveKeyWithValue("test-backend", filepath.Join(tempDir, "test-backend", "run.sh")))
})
})
Describe("DeleteBackendFromSystem", func() {
It("should delete backend directory", func() {
backendName := "test-backend"
backendPath := filepath.Join(tempDir, backendName)
// Create a dummy backend directory
err := os.MkdirAll(backendPath, 0750)
Expect(err).NotTo(HaveOccurred())
err = DeleteBackendFromSystem(tempDir, backendName)
Expect(err).NotTo(HaveOccurred())
Expect(backendPath).NotTo(BeADirectory())
})
It("should not error when backend doesn't exist", func() {
err := DeleteBackendFromSystem(tempDir, "non-existent")
Expect(err).NotTo(HaveOccurred())
})
})
Describe("ListSystemBackends", func() {
It("should list backends without aliases", func() {
// Create some dummy backend directories
backendNames := []string{"backend1", "backend2", "backend3"}
for _, name := range backendNames {
err := os.MkdirAll(filepath.Join(tempDir, name), 0750)
Expect(err).NotTo(HaveOccurred())
}
backends, err := ListSystemBackends(tempDir)
Expect(err).NotTo(HaveOccurred())
Expect(backends).To(HaveLen(len(backendNames)))
for _, name := range backendNames {
Expect(backends).To(HaveKeyWithValue(name, filepath.Join(tempDir, name, "run.sh")))
}
})
It("should handle backends with aliases", func() {
backendName := "backend1"
alias := "alias1"
backendPath := filepath.Join(tempDir, backendName)
// Create backend directory
err := os.MkdirAll(backendPath, 0750)
Expect(err).NotTo(HaveOccurred())
// Create alias file
err = os.WriteFile(filepath.Join(backendPath, "alias"), []byte(alias), 0644)
Expect(err).NotTo(HaveOccurred())
backends, err := ListSystemBackends(tempDir)
Expect(err).NotTo(HaveOccurred())
Expect(backends).To(HaveKeyWithValue(alias, filepath.Join(tempDir, backendName, "run.sh")))
})
It("should return error when base path doesn't exist", func() {
_, err := ListSystemBackends(filepath.Join(tempDir, "non-existent"))
Expect(err).To(HaveOccurred())
})
})
})

273
core/gallery/gallery.go
View file

@ -1,109 +1,109 @@
package gallery
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"dario.cat/mergo"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/rs/zerolog/log"
"gopkg.in/yaml.v2"
)
// Installs a model from the gallery
func InstallModelFromGallery(galleries []config.Gallery, name string, basePath string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan bool) error {
applyModel := func(model *GalleryModel) error {
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
var config Config
if len(model.URL) > 0 {
var err error
config, err = GetGalleryConfigFromURL(model.URL, basePath)
if err != nil {
return err
}
config.Description = model.Description
config.License = model.License
} else if len(model.ConfigFile) > 0 {
// TODO: is this worse than using the override method with a blank cfg yaml?
reYamlConfig, err := yaml.Marshal(model.ConfigFile)
if err != nil {
return err
}
config = Config{
ConfigFile: string(reYamlConfig),
Description: model.Description,
License: model.License,
URLs: model.URLs,
Name: model.Name,
Files: make([]File, 0), // Real values get added below, must be blank
// Prompt Template Skipped for now - I expect in this mode that they will be delivered as files.
}
} else {
return fmt.Errorf("invalid gallery model %+v", model)
}
installName := model.Name
if req.Name != "" {
installName = req.Name
}
// Copy the model configuration from the request schema
config.URLs = append(config.URLs, model.URLs...)
config.Icon = model.Icon
config.Files = append(config.Files, req.AdditionalFiles...)
config.Files = append(config.Files, model.AdditionalFiles...)
// TODO model.Overrides could be merged with user overrides (not defined yet)
if err := mergo.Merge(&model.Overrides, req.Overrides, mergo.WithOverride); err != nil {
return err
}
if err := InstallModel(basePath, installName, &config, model.Overrides, downloadStatus, enforceScan); err != nil {
return err
}
return nil
}
models, err := AvailableGalleryModels(galleries, basePath)
func GetGalleryConfigFromURL[T any](url string, basePath string) (T, error) {
var config T
uri := downloader.URI(url)
err := uri.DownloadWithCallback(basePath, func(url string, d []byte) error {
return yaml.Unmarshal(d, &config)
})
if err != nil {
return err
log.Error().Err(err).Str("url", url).Msg("failed to get gallery config for url")
return config, err
}
model := FindModel(models, name, basePath)
if model == nil {
return fmt.Errorf("no model found with name %q", name)
}
return applyModel(model)
return config, nil
}
func FindModel(models []*GalleryModel, name string, basePath string) *GalleryModel {
var model *GalleryModel
func ReadConfigFile[T any](filePath string) (*T, error) {
// Read the YAML file
yamlFile, err := os.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("failed to read YAML file: %v", err)
}
// Unmarshal YAML data into a Config struct
var config T
err = yaml.Unmarshal(yamlFile, &config)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal YAML: %v", err)
}
return &config, nil
}
type GalleryElement interface {
SetGallery(gallery config.Gallery)
SetInstalled(installed bool)
GetName() string
GetDescription() string
GetTags() []string
GetGallery() config.Gallery
}
type GalleryElements[T GalleryElement] []T
func (gm GalleryElements[T]) Search(term string) GalleryElements[T] {
var filteredModels GalleryElements[T]
for _, m := range gm {
if strings.Contains(m.GetName(), term) ||
strings.Contains(m.GetDescription(), term) ||
strings.Contains(m.GetGallery().Name, term) ||
strings.Contains(strings.Join(m.GetTags(), ","), term) {
filteredModels = append(filteredModels, m)
}
}
return filteredModels
}
func (gm GalleryElements[T]) FindByName(name string) T {
for _, m := range gm {
if strings.EqualFold(m.GetName(), name) {
return m
}
}
var zero T
return zero
}
func (gm GalleryElements[T]) Paginate(pageNum int, itemsNum int) GalleryElements[T] {
start := (pageNum - 1) * itemsNum
end := start + itemsNum
if start > len(gm) {
start = len(gm)
}
if end > len(gm) {
end = len(gm)
}
return gm[start:end]
}
func FindGalleryElement[T GalleryElement](models []T, name string, basePath string) T {
var model T
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
if !strings.Contains(name, "@") {
for _, m := range models {
if strings.EqualFold(m.Name, name) {
if strings.EqualFold(m.GetName(), name) {
model = m
break
}
}
if model == nil {
return nil
}
} else {
for _, m := range models {
if strings.EqualFold(name, fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name)) {
if strings.EqualFold(name, fmt.Sprintf("%s@%s", m.GetGallery().Name, m.GetName())) {
model = m
break
}
@ -116,12 +116,28 @@ func FindModel(models []*GalleryModel, name string, basePath string) *GalleryMod
// List available models
// Models galleries are a list of yaml files that are hosted on a remote server (for example github).
// Each yaml file contains a list of models that can be downloaded and optionally overrides to define a new model setting.
func AvailableGalleryModels(galleries []config.Gallery, basePath string) (GalleryModels, error) {
func AvailableGalleryModels(galleries []config.Gallery, basePath string) (GalleryElements[*GalleryModel], error) {
var models []*GalleryModel
// Get models from galleries
for _, gallery := range galleries {
galleryModels, err := getGalleryModels(gallery, basePath)
galleryModels, err := getGalleryElements[*GalleryModel](gallery, basePath)
if err != nil {
return nil, err
}
models = append(models, galleryModels...)
}
return models, nil
}
// List available backends
func AvailableBackends(galleries []config.Gallery, basePath string) (GalleryElements[*GalleryBackend], error) {
var models []*GalleryBackend
// Get models from galleries
for _, gallery := range galleries {
galleryModels, err := getGalleryElements[*GalleryBackend](gallery, basePath)
if err != nil {
return nil, err
}
@ -146,8 +162,8 @@ func findGalleryURLFromReferenceURL(url string, basePath string) (string, error)
return refFile, err
}
func getGalleryModels(gallery config.Gallery, basePath string) ([]*GalleryModel, error) {
var models []*GalleryModel = []*GalleryModel{}
func getGalleryElements[T GalleryElement](gallery config.Gallery, basePath string) ([]T, error) {
var models []T = []T{}
if strings.HasSuffix(gallery.URL, ".ref") {
var err error
@ -170,97 +186,16 @@ func getGalleryModels(gallery config.Gallery, basePath string) ([]*GalleryModel,
// Add gallery to models
for _, model := range models {
model.Gallery = gallery
model.SetGallery(gallery)
// we check if the model was already installed by checking if the config file exists
// TODO: (what to do if the model doesn't install a config file?)
if _, err := os.Stat(filepath.Join(basePath, fmt.Sprintf("%s.yaml", model.Name))); err == nil {
model.Installed = true
// TODO: This is sub-optimal now that the gallery handles both backends and models - we need to abstract this away
if _, err := os.Stat(filepath.Join(basePath, fmt.Sprintf("%s.yaml", model.GetName()))); err == nil {
model.SetInstalled(true)
}
if _, err := os.Stat(filepath.Join(basePath, model.GetName())); err == nil {
model.SetInstalled(true)
}
}
return models, nil
}
func GetLocalModelConfiguration(basePath string, name string) (*Config, error) {
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
galleryFile := filepath.Join(basePath, galleryFileName(name))
return ReadConfigFile(galleryFile)
}
func DeleteModelFromSystem(basePath string, name string, additionalFiles []string) error {
// os.PathSeparator is not allowed in model names. Replace them with "__" to avoid conflicts with file paths.
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
configFile := filepath.Join(basePath, fmt.Sprintf("%s.yaml", name))
galleryFile := filepath.Join(basePath, galleryFileName(name))
for _, f := range []string{configFile, galleryFile} {
if err := utils.VerifyPath(f, basePath); err != nil {
return fmt.Errorf("failed to verify path %s: %w", f, err)
}
}
var err error
// Delete all the files associated to the model
// read the model config
galleryconfig, err := ReadConfigFile(galleryFile)
if err != nil {
log.Error().Err(err).Msgf("failed to read gallery file %s", configFile)
}
var filesToRemove []string
// Remove additional files
if galleryconfig != nil {
for _, f := range galleryconfig.Files {
fullPath := filepath.Join(basePath, f.Filename)
filesToRemove = append(filesToRemove, fullPath)
}
}
for _, f := range additionalFiles {
fullPath := filepath.Join(filepath.Join(basePath, f))
filesToRemove = append(filesToRemove, fullPath)
}
filesToRemove = append(filesToRemove, configFile)
filesToRemove = append(filesToRemove, galleryFile)
// skip duplicates
filesToRemove = utils.Unique(filesToRemove)
// Removing files
for _, f := range filesToRemove {
if e := os.Remove(f); e != nil {
err = errors.Join(err, fmt.Errorf("failed to remove file %s: %w", f, e))
}
}
return err
}
// This is ***NEVER*** going to be perfect or finished.
// This is a BEST EFFORT function to surface known-vulnerable models to users.
func SafetyScanGalleryModels(galleries []config.Gallery, basePath string) error {
galleryModels, err := AvailableGalleryModels(galleries, basePath)
if err != nil {
return err
}
for _, gM := range galleryModels {
if gM.Installed {
err = errors.Join(err, SafetyScanGalleryModel(gM))
}
}
return err
}
func SafetyScanGalleryModel(galleryModel *GalleryModel) error {
for _, file := range galleryModel.AdditionalFiles {
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
log.Error().Str("model", galleryModel.Name).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("Contains unsafe file(s)!")
return err
}
}
return nil
}

7
core/gallery/gallery_suite_test.go
View file

@ -1,7 +1,6 @@
package gallery_test
import (
"os"
"testing"
. "github.com/onsi/ginkgo/v2"
@ -12,9 +11,3 @@ func TestGallery(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Gallery test suite")
}
var _ = BeforeSuite(func() {
if os.Getenv("FIXTURES") == "" {
Fail("FIXTURES env var not set")
}
})

19
core/gallery/metadata_type.go Normal file
View file

@ -0,0 +1,19 @@
package gallery
import "github.com/mudler/LocalAI/core/config"
type Metadata struct {
URL string `json:"url,omitempty" yaml:"url,omitempty"`
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Description string `json:"description,omitempty" yaml:"description,omitempty"`
License string `json:"license,omitempty" yaml:"license,omitempty"`
URLs []string `json:"urls,omitempty" yaml:"urls,omitempty"`
Icon string `json:"icon,omitempty" yaml:"icon,omitempty"`
Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"`
// AdditionalFiles are used to add additional files to the model
AdditionalFiles []File `json:"files,omitempty" yaml:"files,omitempty"`
// Gallery is a reference to the gallery which contains the model
Gallery config.Gallery `json:"gallery,omitempty" yaml:"gallery,omitempty"`
// Installed is used to indicate if the model is installed or not
Installed bool `json:"installed,omitempty" yaml:"installed,omitempty"`
}

188
core/gallery/models.go
View file

@ -5,8 +5,10 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"dario.cat/mergo"
"github.com/mudler/LocalAI/core/config"
lconfig "github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/downloader"
"github.com/mudler/LocalAI/pkg/utils"
@ -41,10 +43,10 @@ prompt_templates:
content: ""
*/
// Config is the model configuration which contains all the model details
// ModelConfig is the model configuration which contains all the model details
// This configuration is read from the gallery endpoint and is used to download and install the model
// It is the internal structure, separated from the request
type Config struct {
type ModelConfig struct {
Description string `yaml:"description"`
Icon string `yaml:"icon"`
License string `yaml:"license"`
@ -66,37 +68,78 @@ type PromptTemplate struct {
Content string `yaml:"content"`
}
func GetGalleryConfigFromURL(url string, basePath string) (Config, error) {
var config Config
uri := downloader.URI(url)
err := uri.DownloadWithCallback(basePath, func(url string, d []byte) error {
return yaml.Unmarshal(d, &config)
})
if err != nil {
log.Error().Err(err).Str("url", url).Msg("failed to get gallery config for url")
return config, err
// Installs a model from the gallery
func InstallModelFromGallery(galleries []config.Gallery, name string, basePath string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan bool) error {
applyModel := func(model *GalleryModel) error {
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
var config ModelConfig
if len(model.URL) > 0 {
var err error
config, err = GetGalleryConfigFromURL[ModelConfig](model.URL, basePath)
if err != nil {
return err
}
config.Description = model.Description
config.License = model.License
} else if len(model.ConfigFile) > 0 {
// TODO: is this worse than using the override method with a blank cfg yaml?
reYamlConfig, err := yaml.Marshal(model.ConfigFile)
if err != nil {
return err
}
config = ModelConfig{
ConfigFile: string(reYamlConfig),
Description: model.Description,
License: model.License,
URLs: model.URLs,
Name: model.Name,
Files: make([]File, 0), // Real values get added below, must be blank
// Prompt Template Skipped for now - I expect in this mode that they will be delivered as files.
}
} else {
return fmt.Errorf("invalid gallery model %+v", model)
}
installName := model.Name
if req.Name != "" {
installName = req.Name
}
// Copy the model configuration from the request schema
config.URLs = append(config.URLs, model.URLs...)
config.Icon = model.Icon
config.Files = append(config.Files, req.AdditionalFiles...)
config.Files = append(config.Files, model.AdditionalFiles...)
// TODO model.Overrides could be merged with user overrides (not defined yet)
if err := mergo.Merge(&model.Overrides, req.Overrides, mergo.WithOverride); err != nil {
return err
}
if err := InstallModel(basePath, installName, &config, model.Overrides, downloadStatus, enforceScan); err != nil {
return err
}
return nil
}
return config, nil
models, err := AvailableGalleryModels(galleries, basePath)
if err != nil {
return err
}
model := FindGalleryElement(models, name, basePath)
if model == nil {
return fmt.Errorf("no model found with name %q", name)
}
return applyModel(model)
}
func ReadConfigFile(filePath string) (*Config, error) {
// Read the YAML file
yamlFile, err := os.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("failed to read YAML file: %v", err)
}
// Unmarshal YAML data into a Config struct
var config Config
err = yaml.Unmarshal(yamlFile, &config)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal YAML: %v", err)
}
return &config, nil
}
func InstallModel(basePath, nameOverride string, config *Config, configOverrides map[string]interface{}, downloadStatus func(string, string, string, float64), enforceScan bool) error {
func InstallModel(basePath, nameOverride string, config *ModelConfig, configOverrides map[string]interface{}, downloadStatus func(string, string, string, float64), enforceScan bool) error {
// Create base path if it doesn't exist
err := os.MkdirAll(basePath, 0750)
if err != nil {
@ -219,3 +262,88 @@ func InstallModel(basePath, nameOverride string, config *Config, configOverrides
func galleryFileName(name string) string {
return "._gallery_" + name + ".yaml"
}
func GetLocalModelConfiguration(basePath string, name string) (*ModelConfig, error) {
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
galleryFile := filepath.Join(basePath, galleryFileName(name))
return ReadConfigFile[ModelConfig](galleryFile)
}
func DeleteModelFromSystem(basePath string, name string, additionalFiles []string) error {
// os.PathSeparator is not allowed in model names. Replace them with "__" to avoid conflicts with file paths.
name = strings.ReplaceAll(name, string(os.PathSeparator), "__")
configFile := filepath.Join(basePath, fmt.Sprintf("%s.yaml", name))
galleryFile := filepath.Join(basePath, galleryFileName(name))
for _, f := range []string{configFile, galleryFile} {
if err := utils.VerifyPath(f, basePath); err != nil {
return fmt.Errorf("failed to verify path %s: %w", f, err)
}
}
var err error
// Delete all the files associated to the model
// read the model config
galleryconfig, err := ReadConfigFile[ModelConfig](galleryFile)
if err != nil {
log.Error().Err(err).Msgf("failed to read gallery file %s", configFile)
}
var filesToRemove []string
// Remove additional files
if galleryconfig != nil {
for _, f := range galleryconfig.Files {
fullPath := filepath.Join(basePath, f.Filename)
filesToRemove = append(filesToRemove, fullPath)
}
}
for _, f := range additionalFiles {
fullPath := filepath.Join(filepath.Join(basePath, f))
filesToRemove = append(filesToRemove, fullPath)
}
filesToRemove = append(filesToRemove, configFile)
filesToRemove = append(filesToRemove, galleryFile)
// skip duplicates
filesToRemove = utils.Unique(filesToRemove)
// Removing files
for _, f := range filesToRemove {
if e := os.Remove(f); e != nil {
err = errors.Join(err, fmt.Errorf("failed to remove file %s: %w", f, e))
}
}
return err
}
// This is ***NEVER*** going to be perfect or finished.
// This is a BEST EFFORT function to surface known-vulnerable models to users.
func SafetyScanGalleryModels(galleries []config.Gallery, basePath string) error {
galleryModels, err := AvailableGalleryModels(galleries, basePath)
if err != nil {
return err
}
for _, gM := range galleryModels {
if gM.Installed {
err = errors.Join(err, SafetyScanGalleryModel(gM))
}
}
return err
}
func SafetyScanGalleryModel(galleryModel *GalleryModel) error {
for _, file := range galleryModel.AdditionalFiles {
scanResults, err := downloader.HuggingFaceScan(downloader.URI(file.URI))
if err != nil && errors.Is(err, downloader.ErrUnsafeFilesFound) {
log.Error().Str("model", galleryModel.Name).Strs("clamAV", scanResults.ClamAVInfectedFiles).Strs("pickles", scanResults.DangerousPickles).Msg("Contains unsafe file(s)!")
return err
}
}
return nil
}

14
core/gallery/models_test.go
View file

@ -16,12 +16,18 @@ const bertEmbeddingsURL = `https://gist.githubusercontent.com/mudler/0a080b166b8
var _ = Describe("Model test", func() {
BeforeEach(func() {
if os.Getenv("FIXTURES") == "" {
Skip("FIXTURES env var not set, skipping model tests")
}
})
Context("Downloading", func() {
It("applies model correctly", func() {
tempdir, err := os.MkdirTemp("", "test")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tempdir)
c, err := ReadConfigFile(filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
c, err := ReadConfigFile[ModelConfig](filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
Expect(err).ToNot(HaveOccurred())
err = InstallModel(tempdir, "", c, map[string]interface{}{}, func(string, string, string, float64) {}, true)
Expect(err).ToNot(HaveOccurred())
@ -107,7 +113,7 @@ var _ = Describe("Model test", func() {
tempdir, err := os.MkdirTemp("", "test")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tempdir)
c, err := ReadConfigFile(filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
c, err := ReadConfigFile[ModelConfig](filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
Expect(err).ToNot(HaveOccurred())
err = InstallModel(tempdir, "foo", c, map[string]interface{}{}, func(string, string, string, float64) {}, true)
@ -123,7 +129,7 @@ var _ = Describe("Model test", func() {
tempdir, err := os.MkdirTemp("", "test")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tempdir)
c, err := ReadConfigFile(filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
c, err := ReadConfigFile[ModelConfig](filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
Expect(err).ToNot(HaveOccurred())
err = InstallModel(tempdir, "foo", c, map[string]interface{}{"backend": "foo"}, func(string, string, string, float64) {}, true)
@ -149,7 +155,7 @@ var _ = Describe("Model test", func() {
tempdir, err := os.MkdirTemp("", "test")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tempdir)
c, err := ReadConfigFile(filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
c, err := ReadConfigFile[ModelConfig](filepath.Join(os.Getenv("FIXTURES"), "gallery_simple.yaml"))
Expect(err).ToNot(HaveOccurred())
err = InstallModel(tempdir, "../../../foo", c, map[string]interface{}{}, func(string, string, string, float64) {}, true)

46
core/gallery/models_types.go Normal file
View file

@ -0,0 +1,46 @@
package gallery
import (
"fmt"
"github.com/mudler/LocalAI/core/config"
)
// GalleryModel is the struct used to represent a model in the gallery returned by the endpoint.
// It is used to install the model by resolving the URL and downloading the files.
// The other fields are used to override the configuration of the model.
type GalleryModel struct {
Metadata `json:",inline" yaml:",inline"`
// config_file is read in the situation where URL is blank - and therefore this is a base config.
ConfigFile map[string]interface{} `json:"config_file,omitempty" yaml:"config_file,omitempty"`
// Overrides are used to override the configuration of the model located at URL
Overrides map[string]interface{} `json:"overrides,omitempty" yaml:"overrides,omitempty"`
}
func (m *GalleryModel) SetGallery(gallery config.Gallery) {
m.Gallery = gallery
}
func (m *GalleryModel) SetInstalled(installed bool) {
m.Installed = installed
}
func (m *GalleryModel) GetName() string {
return m.Name
}
func (m *GalleryModel) GetGallery() config.Gallery {
return m.Gallery
}
func (m GalleryModel) ID() string {
return fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name)
}
func (m *GalleryModel) GetTags() []string {
return m.Tags
}
func (m *GalleryModel) GetDescription() string {
return m.Description
}

25
core/gallery/op.go
View file

@ -1,25 +0,0 @@
package gallery
import "github.com/mudler/LocalAI/core/config"
type GalleryOp struct {
Id string
GalleryModelName string
ConfigURL string
Delete bool
Req GalleryModel
Galleries []config.Gallery
}
type GalleryOpStatus struct {
Deletion bool `json:"deletion"` // Deletion is true if the operation is a deletion
FileName string `json:"file_name"`
Error error `json:"error"`
Processed bool `json:"processed"`
Message string `json:"message"`
Progress float64 `json:"progress"`
TotalFileSize string `json:"file_size"`
DownloadedFileSize string `json:"downloaded_size"`
GalleryModelName string `json:"gallery_model_name"`
}

76
core/gallery/request.go
View file

@ -1,76 +0,0 @@
package gallery
import (
"fmt"
"strings"
"github.com/mudler/LocalAI/core/config"
)
// GalleryModel is the struct used to represent a model in the gallery returned by the endpoint.
// It is used to install the model by resolving the URL and downloading the files.
// The other fields are used to override the configuration of the model.
type GalleryModel struct {
Metadata `json:",inline" yaml:",inline"`
// config_file is read in the situation where URL is blank - and therefore this is a base config.
ConfigFile map[string]interface{} `json:"config_file,omitempty" yaml:"config_file,omitempty"`
// Overrides are used to override the configuration of the model located at URL
Overrides map[string]interface{} `json:"overrides,omitempty" yaml:"overrides,omitempty"`
}
type Metadata struct {
URL string `json:"url,omitempty" yaml:"url,omitempty"`
Name string `json:"name,omitempty" yaml:"name,omitempty"`
Description string `json:"description,omitempty" yaml:"description,omitempty"`
License string `json:"license,omitempty" yaml:"license,omitempty"`
URLs []string `json:"urls,omitempty" yaml:"urls,omitempty"`
Icon string `json:"icon,omitempty" yaml:"icon,omitempty"`
Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"`
// AdditionalFiles are used to add additional files to the model
AdditionalFiles []File `json:"files,omitempty" yaml:"files,omitempty"`
// Gallery is a reference to the gallery which contains the model
Gallery config.Gallery `json:"gallery,omitempty" yaml:"gallery,omitempty"`
// Installed is used to indicate if the model is installed or not
Installed bool `json:"installed,omitempty" yaml:"installed,omitempty"`
}
func (m GalleryModel) ID() string {
return fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name)
}
type GalleryModels []*GalleryModel
func (gm GalleryModels) Search(term string) GalleryModels {
var filteredModels GalleryModels
for _, m := range gm {
if strings.Contains(m.Name, term) ||
strings.Contains(m.Description, term) ||
strings.Contains(m.Gallery.Name, term) ||
strings.Contains(strings.Join(m.Tags, ","), term) {
filteredModels = append(filteredModels, m)
}
}
return filteredModels
}
func (gm GalleryModels) FindByName(name string) *GalleryModel {
for _, m := range gm {
if strings.EqualFold(m.Name, name) {
return m
}
}
return nil
}
func (gm GalleryModels) Paginate(pageNum int, itemsNum int) GalleryModels {
start := (pageNum - 1) * itemsNum
end := start + itemsNum
if start > len(gm) {
start = len(gm)
}
if end > len(gm) {
end = len(gm)
}
return gm[start:end]
}

2
core/gallery/request_test.go
View file

@ -14,7 +14,7 @@ var _ = Describe("Gallery API tests", func() {
URL: "github:go-skynet/model-gallery/gpt4all-j.yaml@main",
},
}
e, err := GetGalleryConfigFromURL(req.URL, "")
e, err := GetGalleryConfigFromURL[ModelConfig](req.URL, "")
Expect(err).ToNot(HaveOccurred())
Expect(e.Name).To(Equal("gpt4all-j"))
})