mirror of
https://github.com/mudler/LocalAI.git
synced 2025-06-17 08:15:00 +00:00
feat: Add backend gallery (#5607)
* feat: Add backend gallery This PR add support to manage backends as similar to models. There is now available a backend gallery which can be used to install and remove extra backends. The backend gallery can be configured similarly as a model gallery, and API calls allows to install and remove new backends in runtime, and as well during the startup phase of LocalAI. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add backends docs Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * wip: Backend Dockerfile for python backends Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat: drop extras images, build python backends separately Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixup on all backends Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * test CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Tweaks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Drop old backends leftovers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixup CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Move dockerfile upper Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fix proto Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Feature dropped for consistency - we prefer model galleries Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add missing packages in the build image Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * exllama is ponly available on cublas Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * pin torch on chatterbox Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixups to index Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Debug CI * Install accellerators deps Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add target arch * Add cuda minor version Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Use self-hosted runners Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: use quay for test images Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixups for vllm and chatterbox Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Small fixups on CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chatterbox is only available for nvidia Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Simplify CI builds Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Adapt test, use qwen3 Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chore(model gallery): add jina-reranker-v1-tiny-en-gguf Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(gguf-parser): recover from potential panics that can happen while reading ggufs with gguf-parser Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Use reranker from llama.cpp in AIO images Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Limit concurrent jobs Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>
This commit is contained in:
Ettore Di Giacinto
GitHub
parent
a7a6020328
commit
2d64269763
114 changed files with 3996 additions and 1382 deletions
|
|
@ -30,10 +30,16 @@ func RegisterLocalAIRoutes(router *fiber.App,
|
|||
|
||||
router.Get("/models/available", modelGalleryEndpointService.ListModelFromGalleryEndpoint())
|
||||
router.Get("/models/galleries", modelGalleryEndpointService.ListModelGalleriesEndpoint())
|
||||
router.Post("/models/galleries", modelGalleryEndpointService.AddModelGalleryEndpoint())
|
||||
router.Delete("/models/galleries", modelGalleryEndpointService.RemoveModelGalleryEndpoint())
|
||||
router.Get("/models/jobs/:uuid", modelGalleryEndpointService.GetOpStatusEndpoint())
|
||||
router.Get("/models/jobs", modelGalleryEndpointService.GetAllStatusEndpoint())
|
||||
|
||||
backendGalleryEndpointService := localai.CreateBackendEndpointService(appConfig.BackendGalleries, appConfig.BackendsPath, galleryService)
|
||||
router.Post("/backends/apply", backendGalleryEndpointService.ApplyBackendEndpoint())
|
||||
router.Post("/backends/delete/:name", backendGalleryEndpointService.DeleteBackendEndpoint())
|
||||
router.Get("/backends", backendGalleryEndpointService.ListBackendsEndpoint())
|
||||
router.Get("/backends/available", backendGalleryEndpointService.ListAvailableBackendsEndpoint())
|
||||
router.Get("/backends/galleries", backendGalleryEndpointService.ListBackendGalleriesEndpoint())
|
||||
router.Get("/backends/jobs/:uuid", backendGalleryEndpointService.GetOpStatusEndpoint())
|
||||
}
|
||||
|
||||
router.Post("/tts",
|
||||
|
|
|
|||
|
|
@ -1,13 +1,6 @@
|
|||
package routes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/core/http/elements"
|
||||
|
|
@ -17,78 +10,20 @@ import (
|
|||
"github.com/mudler/LocalAI/core/services"
|
||||
"github.com/mudler/LocalAI/internal"
|
||||
"github.com/mudler/LocalAI/pkg/model"
|
||||
"github.com/mudler/LocalAI/pkg/xsync"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/google/uuid"
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
type modelOpCache struct {
|
||||
status *xsync.SyncedMap[string, string]
|
||||
}
|
||||
|
||||
func NewModelOpCache() *modelOpCache {
|
||||
return &modelOpCache{
|
||||
status: xsync.NewSyncedMap[string, string](),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *modelOpCache) Set(key string, value string) {
|
||||
m.status.Set(key, value)
|
||||
}
|
||||
|
||||
func (m *modelOpCache) Get(key string) string {
|
||||
return m.status.Get(key)
|
||||
}
|
||||
|
||||
func (m *modelOpCache) DeleteUUID(uuid string) {
|
||||
for _, k := range m.status.Keys() {
|
||||
if m.status.Get(k) == uuid {
|
||||
m.status.Delete(k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *modelOpCache) Map() map[string]string {
|
||||
return m.status.Map()
|
||||
}
|
||||
|
||||
func (m *modelOpCache) Exists(key string) bool {
|
||||
return m.status.Exists(key)
|
||||
}
|
||||
|
||||
func RegisterUIRoutes(app *fiber.App,
|
||||
cl *config.BackendConfigLoader,
|
||||
ml *model.ModelLoader,
|
||||
appConfig *config.ApplicationConfig,
|
||||
galleryService *services.GalleryService) {
|
||||
|
||||
// keeps the state of models that are being installed from the UI
|
||||
var processingModels = NewModelOpCache()
|
||||
// keeps the state of ops that are started from the UI
|
||||
var processingOps = services.NewOpCache(galleryService)
|
||||
|
||||
// modelStatus returns the current status of the models being processed (installation or deletion)
|
||||
// it is called asynchronously from the UI
|
||||
modelStatus := func() (map[string]string, map[string]string) {
|
||||
processingModelsData := processingModels.Map()
|
||||
|
||||
taskTypes := map[string]string{}
|
||||
|
||||
for k, v := range processingModelsData {
|
||||
status := galleryService.GetStatus(v)
|
||||
taskTypes[k] = "Installation"
|
||||
if status != nil && status.Deletion {
|
||||
taskTypes[k] = "Deletion"
|
||||
} else if status == nil {
|
||||
taskTypes[k] = "Waiting"
|
||||
}
|
||||
}
|
||||
|
||||
return processingModelsData, taskTypes
|
||||
}
|
||||
|
||||
app.Get("/", localai.WelcomeEndpoint(appConfig, cl, ml, modelStatus))
|
||||
app.Get("/", localai.WelcomeEndpoint(appConfig, cl, ml, processingOps))
|
||||
|
||||
if p2p.IsP2PEnabled() {
|
||||
app.Get("/p2p", func(c *fiber.Ctx) error {
|
||||
|
|
@ -124,262 +59,8 @@ func RegisterUIRoutes(app *fiber.App,
|
|||
}
|
||||
|
||||
if !appConfig.DisableGalleryEndpoint {
|
||||
|
||||
// Show the Models page (all models)
|
||||
app.Get("/browse", func(c *fiber.Ctx) error {
|
||||
term := c.Query("term")
|
||||
page := c.Query("page")
|
||||
items := c.Query("items")
|
||||
|
||||
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.ModelPath)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("could not list models from galleries")
|
||||
return c.Status(fiber.StatusInternalServerError).Render("views/error", fiber.Map{
|
||||
"Title": "LocalAI - Models",
|
||||
"BaseURL": utils.BaseURL(c),
|
||||
"Version": internal.PrintableVersion(),
|
||||
"ErrorCode": "500",
|
||||
"ErrorMessage": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Get all available tags
|
||||
allTags := map[string]struct{}{}
|
||||
tags := []string{}
|
||||
for _, m := range models {
|
||||
for _, t := range m.Tags {
|
||||
allTags[t] = struct{}{}
|
||||
}
|
||||
}
|
||||
for t := range allTags {
|
||||
tags = append(tags, t)
|
||||
}
|
||||
sort.Strings(tags)
|
||||
|
||||
if term != "" {
|
||||
models = gallery.GalleryModels(models).Search(term)
|
||||
}
|
||||
|
||||
// Get model statuses
|
||||
processingModelsData, taskTypes := modelStatus()
|
||||
|
||||
summary := fiber.Map{
|
||||
"Title": "LocalAI - Models",
|
||||
"BaseURL": utils.BaseURL(c),
|
||||
"Version": internal.PrintableVersion(),
|
||||
"Models": template.HTML(elements.ListModels(models, processingModels, galleryService)),
|
||||
"Repositories": appConfig.Galleries,
|
||||
"AllTags": tags,
|
||||
"ProcessingModels": processingModelsData,
|
||||
"AvailableModels": len(models),
|
||||
"IsP2PEnabled": p2p.IsP2PEnabled(),
|
||||
|
||||
"TaskTypes": taskTypes,
|
||||
// "ApplicationConfig": appConfig,
|
||||
}
|
||||
|
||||
if page == "" {
|
||||
page = "1"
|
||||
}
|
||||
|
||||
if page != "" {
|
||||
// return a subset of the models
|
||||
pageNum, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
return c.Status(fiber.StatusBadRequest).SendString("Invalid page number")
|
||||
}
|
||||
|
||||
if pageNum == 0 {
|
||||
return c.Render("views/models", summary)
|
||||
}
|
||||
|
||||
itemsNum, err := strconv.Atoi(items)
|
||||
if err != nil {
|
||||
itemsNum = 21
|
||||
}
|
||||
|
||||
totalPages := int(math.Ceil(float64(len(models)) / float64(itemsNum)))
|
||||
|
||||
models = models.Paginate(pageNum, itemsNum)
|
||||
|
||||
prevPage := pageNum - 1
|
||||
nextPage := pageNum + 1
|
||||
if prevPage < 1 {
|
||||
prevPage = 1
|
||||
}
|
||||
if nextPage > totalPages {
|
||||
nextPage = totalPages
|
||||
}
|
||||
if prevPage != pageNum {
|
||||
summary["PrevPage"] = prevPage
|
||||
}
|
||||
summary["NextPage"] = nextPage
|
||||
summary["TotalPages"] = totalPages
|
||||
summary["CurrentPage"] = pageNum
|
||||
summary["Models"] = template.HTML(elements.ListModels(models, processingModels, galleryService))
|
||||
}
|
||||
|
||||
// Render index
|
||||
return c.Render("views/models", summary)
|
||||
})
|
||||
|
||||
// Show the models, filtered from the user input
|
||||
// https://htmx.org/examples/active-search/
|
||||
app.Post("/browse/search/models", func(c *fiber.Ctx) error {
|
||||
page := c.Query("page")
|
||||
items := c.Query("items")
|
||||
|
||||
form := struct {
|
||||
Search string `form:"search"`
|
||||
}{}
|
||||
if err := c.BodyParser(&form); err != nil {
|
||||
return c.Status(fiber.StatusBadRequest).SendString(bluemonday.StrictPolicy().Sanitize(err.Error()))
|
||||
}
|
||||
|
||||
models, _ := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.ModelPath)
|
||||
|
||||
if page != "" {
|
||||
// return a subset of the models
|
||||
pageNum, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
return c.Status(fiber.StatusBadRequest).SendString("Invalid page number")
|
||||
}
|
||||
|
||||
itemsNum, err := strconv.Atoi(items)
|
||||
if err != nil {
|
||||
itemsNum = 21
|
||||
}
|
||||
|
||||
models = models.Paginate(pageNum, itemsNum)
|
||||
}
|
||||
|
||||
if form.Search != "" {
|
||||
models = models.Search(form.Search)
|
||||
}
|
||||
|
||||
return c.SendString(elements.ListModels(models, processingModels, galleryService))
|
||||
})
|
||||
|
||||
/*
|
||||
|
||||
Install routes
|
||||
|
||||
*/
|
||||
|
||||
// This route is used when the "Install" button is pressed, we submit here a new job to the gallery service
|
||||
// https://htmx.org/examples/progress-bar/
|
||||
app.Post("/browse/install/model/:id", func(c *fiber.Ctx) error {
|
||||
galleryID := strings.Clone(c.Params("id")) // note: strings.Clone is required for multiple requests!
|
||||
log.Debug().Msgf("UI job submitted to install : %+v\n", galleryID)
|
||||
|
||||
id, err := uuid.NewUUID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uid := id.String()
|
||||
|
||||
processingModels.Set(galleryID, uid)
|
||||
|
||||
op := gallery.GalleryOp{
|
||||
Id: uid,
|
||||
GalleryModelName: galleryID,
|
||||
Galleries: appConfig.Galleries,
|
||||
}
|
||||
go func() {
|
||||
galleryService.C <- op
|
||||
}()
|
||||
|
||||
return c.SendString(elements.StartProgressBar(uid, "0", "Installation"))
|
||||
})
|
||||
|
||||
// This route is used when the "Install" button is pressed, we submit here a new job to the gallery service
|
||||
// https://htmx.org/examples/progress-bar/
|
||||
app.Post("/browse/delete/model/:id", func(c *fiber.Ctx) error {
|
||||
galleryID := strings.Clone(c.Params("id")) // note: strings.Clone is required for multiple requests!
|
||||
log.Debug().Msgf("UI job submitted to delete : %+v\n", galleryID)
|
||||
var galleryName = galleryID
|
||||
if strings.Contains(galleryID, "@") {
|
||||
// if the galleryID contains a @ it means that it's a model from a gallery
|
||||
// but we want to delete it from the local models which does not need
|
||||
// a repository ID
|
||||
galleryName = strings.Split(galleryID, "@")[1]
|
||||
}
|
||||
|
||||
id, err := uuid.NewUUID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uid := id.String()
|
||||
|
||||
// Track the deletion job by galleryID and galleryName
|
||||
// The GalleryID contains information about the repository,
|
||||
// while the GalleryName is ONLY the name of the model
|
||||
processingModels.Set(galleryName, uid)
|
||||
processingModels.Set(galleryID, uid)
|
||||
|
||||
op := gallery.GalleryOp{
|
||||
Id: uid,
|
||||
Delete: true,
|
||||
GalleryModelName: galleryName,
|
||||
}
|
||||
go func() {
|
||||
galleryService.C <- op
|
||||
cl.RemoveBackendConfig(galleryName)
|
||||
}()
|
||||
|
||||
return c.SendString(elements.StartProgressBar(uid, "0", "Deletion"))
|
||||
})
|
||||
|
||||
// Display the job current progress status
|
||||
// If the job is done, we trigger the /browse/job/:uid route
|
||||
// https://htmx.org/examples/progress-bar/
|
||||
app.Get("/browse/job/progress/:uid", func(c *fiber.Ctx) error {
|
||||
jobUID := strings.Clone(c.Params("uid")) // note: strings.Clone is required for multiple requests!
|
||||
|
||||
status := galleryService.GetStatus(jobUID)
|
||||
if status == nil {
|
||||
//fmt.Errorf("could not find any status for ID")
|
||||
return c.SendString(elements.ProgressBar("0"))
|
||||
}
|
||||
|
||||
if status.Progress == 100 {
|
||||
c.Set("HX-Trigger", "done") // this triggers /browse/job/:uid (which is when the job is done)
|
||||
return c.SendString(elements.ProgressBar("100"))
|
||||
}
|
||||
if status.Error != nil {
|
||||
// TODO: instead of deleting the job, we should keep it in the cache and make it dismissable by the user
|
||||
processingModels.DeleteUUID(jobUID)
|
||||
return c.SendString(elements.ErrorProgress(status.Error.Error(), status.GalleryModelName))
|
||||
}
|
||||
|
||||
return c.SendString(elements.ProgressBar(fmt.Sprint(status.Progress)))
|
||||
})
|
||||
|
||||
// this route is hit when the job is done, and we display the
|
||||
// final state (for now just displays "Installation completed")
|
||||
app.Get("/browse/job/:uid", func(c *fiber.Ctx) error {
|
||||
jobUID := strings.Clone(c.Params("uid")) // note: strings.Clone is required for multiple requests!
|
||||
|
||||
status := galleryService.GetStatus(jobUID)
|
||||
|
||||
galleryID := ""
|
||||
processingModels.DeleteUUID(jobUID)
|
||||
if galleryID == "" {
|
||||
log.Debug().Msgf("no processing model found for job : %+v\n", jobUID)
|
||||
}
|
||||
|
||||
log.Debug().Msgf("JOB finished : %+v\n", status)
|
||||
showDelete := true
|
||||
displayText := "Installation completed"
|
||||
if status.Deletion {
|
||||
showDelete = false
|
||||
displayText = "Deletion completed"
|
||||
}
|
||||
|
||||
return c.SendString(elements.DoneProgress(galleryID, displayText, showDelete))
|
||||
})
|
||||
registerGalleryRoutes(app, cl, appConfig, galleryService, processingOps)
|
||||
registerBackendGalleryRoutes(app, appConfig, galleryService, processingOps)
|
||||
}
|
||||
|
||||
app.Get("/talk/", func(c *fiber.Ctx) error {
|
||||
|
|
@ -412,7 +93,7 @@ func RegisterUIRoutes(app *fiber.App,
|
|||
return c.Redirect(utils.BaseURL(c))
|
||||
}
|
||||
modelThatCanBeUsed := ""
|
||||
galleryConfigs := map[string]*gallery.Config{}
|
||||
galleryConfigs := map[string]*gallery.ModelConfig{}
|
||||
|
||||
for _, m := range backendConfigs {
|
||||
cfg, err := gallery.GetLocalModelConfiguration(ml.ModelPath, m.Name)
|
||||
|
|
@ -452,7 +133,7 @@ func RegisterUIRoutes(app *fiber.App,
|
|||
backendConfigs := cl.GetAllBackendConfigs()
|
||||
modelsWithoutConfig, _ := services.ListModels(cl, ml, config.NoFilterFn, services.LOOSE_ONLY)
|
||||
|
||||
galleryConfigs := map[string]*gallery.Config{}
|
||||
galleryConfigs := map[string]*gallery.ModelConfig{}
|
||||
|
||||
for _, m := range backendConfigs {
|
||||
cfg, err := gallery.GetLocalModelConfiguration(ml.ModelPath, m.Name)
|
||||
|
|
|
|||
258
core/http/routes/ui_backend_gallery.go
Normal file
258
core/http/routes/ui_backend_gallery.go
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
package routes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/google/uuid"
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/core/http/elements"
|
||||
"github.com/mudler/LocalAI/core/http/utils"
|
||||
"github.com/mudler/LocalAI/core/services"
|
||||
"github.com/mudler/LocalAI/internal"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func registerBackendGalleryRoutes(app *fiber.App, appConfig *config.ApplicationConfig, galleryService *services.GalleryService, opcache *services.OpCache) {
|
||||
|
||||
// Show the Backends page (all backends)
|
||||
app.Get("/browse/backends", func(c *fiber.Ctx) error {
|
||||
term := c.Query("term")
|
||||
page := c.Query("page")
|
||||
items := c.Query("items")
|
||||
|
||||
backends, err := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.BackendsPath)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("could not list backends from galleries")
|
||||
return c.Status(fiber.StatusInternalServerError).Render("views/error", fiber.Map{
|
||||
"Title": "LocalAI - Backends",
|
||||
"BaseURL": utils.BaseURL(c),
|
||||
"Version": internal.PrintableVersion(),
|
||||
"ErrorCode": "500",
|
||||
"ErrorMessage": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Get all available tags
|
||||
allTags := map[string]struct{}{}
|
||||
tags := []string{}
|
||||
for _, b := range backends {
|
||||
for _, t := range b.Tags {
|
||||
allTags[t] = struct{}{}
|
||||
}
|
||||
}
|
||||
for t := range allTags {
|
||||
tags = append(tags, t)
|
||||
}
|
||||
sort.Strings(tags)
|
||||
|
||||
if term != "" {
|
||||
backends = gallery.GalleryElements[*gallery.GalleryBackend](backends).Search(term)
|
||||
}
|
||||
|
||||
// Get backend statuses
|
||||
processingBackendsData, taskTypes := opcache.GetStatus()
|
||||
|
||||
summary := fiber.Map{
|
||||
"Title": "LocalAI - Backends",
|
||||
"BaseURL": utils.BaseURL(c),
|
||||
"Version": internal.PrintableVersion(),
|
||||
"Backends": template.HTML(elements.ListBackends(backends, opcache, galleryService)),
|
||||
"Repositories": appConfig.BackendGalleries,
|
||||
"AllTags": tags,
|
||||
"ProcessingBackends": processingBackendsData,
|
||||
"AvailableBackends": len(backends),
|
||||
"TaskTypes": taskTypes,
|
||||
}
|
||||
|
||||
if page == "" {
|
||||
page = "1"
|
||||
}
|
||||
|
||||
if page != "" {
|
||||
// return a subset of the backends
|
||||
pageNum, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
return c.Status(fiber.StatusBadRequest).SendString("Invalid page number")
|
||||
}
|
||||
|
||||
if pageNum == 0 {
|
||||
return c.Render("views/backends", summary)
|
||||
}
|
||||
|
||||
itemsNum, err := strconv.Atoi(items)
|
||||
if err != nil {
|
||||
itemsNum = 21
|
||||
}
|
||||
|
||||
totalPages := int(math.Ceil(float64(len(backends)) / float64(itemsNum)))
|
||||
|
||||
backends = backends.Paginate(pageNum, itemsNum)
|
||||
|
||||
prevPage := pageNum - 1
|
||||
nextPage := pageNum + 1
|
||||
if prevPage < 1 {
|
||||
prevPage = 1
|
||||
}
|
||||
if nextPage > totalPages {
|
||||
nextPage = totalPages
|
||||
}
|
||||
if prevPage != pageNum {
|
||||
summary["PrevPage"] = prevPage
|
||||
}
|
||||
summary["NextPage"] = nextPage
|
||||
summary["TotalPages"] = totalPages
|
||||
summary["CurrentPage"] = pageNum
|
||||
summary["Backends"] = template.HTML(elements.ListBackends(backends, opcache, galleryService))
|
||||
}
|
||||
|
||||
// Render index
|
||||
return c.Render("views/backends", summary)
|
||||
})
|
||||
|
||||
// Show the backends, filtered from the user input
|
||||
app.Post("/browse/search/backends", func(c *fiber.Ctx) error {
|
||||
page := c.Query("page")
|
||||
items := c.Query("items")
|
||||
|
||||
form := struct {
|
||||
Search string `form:"search"`
|
||||
}{}
|
||||
if err := c.BodyParser(&form); err != nil {
|
||||
return c.Status(fiber.StatusBadRequest).SendString(bluemonday.StrictPolicy().Sanitize(err.Error()))
|
||||
}
|
||||
|
||||
backends, _ := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.BackendsPath)
|
||||
|
||||
if page != "" {
|
||||
// return a subset of the backends
|
||||
pageNum, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
return c.Status(fiber.StatusBadRequest).SendString("Invalid page number")
|
||||
}
|
||||
|
||||
itemsNum, err := strconv.Atoi(items)
|
||||
if err != nil {
|
||||
itemsNum = 21
|
||||
}
|
||||
|
||||
backends = backends.Paginate(pageNum, itemsNum)
|
||||
}
|
||||
|
||||
if form.Search != "" {
|
||||
backends = backends.Search(form.Search)
|
||||
}
|
||||
|
||||
return c.SendString(elements.ListBackends(backends, opcache, galleryService))
|
||||
})
|
||||
|
||||
// Install backend route
|
||||
app.Post("/browse/install/backend/:id", func(c *fiber.Ctx) error {
|
||||
backendID := strings.Clone(c.Params("id")) // note: strings.Clone is required for multiple requests!
|
||||
log.Debug().Msgf("UI job submitted to install backend: %+v\n", backendID)
|
||||
|
||||
id, err := uuid.NewUUID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uid := id.String()
|
||||
|
||||
opcache.Set(backendID, uid)
|
||||
|
||||
op := services.GalleryOp[gallery.GalleryBackend]{
|
||||
ID: uid,
|
||||
GalleryElementName: backendID,
|
||||
Galleries: appConfig.BackendGalleries,
|
||||
}
|
||||
go func() {
|
||||
galleryService.BackendGalleryChannel <- op
|
||||
}()
|
||||
|
||||
return c.SendString(elements.StartBackendProgressBar(uid, "0", "Backend Installation"))
|
||||
})
|
||||
|
||||
// Delete backend route
|
||||
app.Post("/browse/delete/backend/:id", func(c *fiber.Ctx) error {
|
||||
backendID := strings.Clone(c.Params("id")) // note: strings.Clone is required for multiple requests!
|
||||
log.Debug().Msgf("UI job submitted to delete backend: %+v\n", backendID)
|
||||
var backendName = backendID
|
||||
if strings.Contains(backendID, "@") {
|
||||
// TODO: this is ugly workaround - we should handle this consistently across the codebase
|
||||
backendName = strings.Split(backendID, "@")[1]
|
||||
}
|
||||
|
||||
id, err := uuid.NewUUID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uid := id.String()
|
||||
|
||||
opcache.Set(backendName, uid)
|
||||
opcache.Set(backendID, uid)
|
||||
|
||||
op := services.GalleryOp[gallery.GalleryBackend]{
|
||||
ID: uid,
|
||||
Delete: true,
|
||||
GalleryElementName: backendName,
|
||||
Galleries: appConfig.BackendGalleries,
|
||||
}
|
||||
go func() {
|
||||
galleryService.BackendGalleryChannel <- op
|
||||
}()
|
||||
|
||||
return c.SendString(elements.StartBackendProgressBar(uid, "0", "Backend Deletion"))
|
||||
})
|
||||
|
||||
// Display the job current progress status
|
||||
app.Get("/browse/backend/job/progress/:uid", func(c *fiber.Ctx) error {
|
||||
jobUID := strings.Clone(c.Params("uid")) // note: strings.Clone is required for multiple requests!
|
||||
|
||||
status := galleryService.GetStatus(jobUID)
|
||||
if status == nil {
|
||||
return c.SendString(elements.ProgressBar("0"))
|
||||
}
|
||||
|
||||
if status.Progress == 100 {
|
||||
c.Set("HX-Trigger", "done") // this triggers /browse/backend/job/:uid
|
||||
return c.SendString(elements.ProgressBar("100"))
|
||||
}
|
||||
if status.Error != nil {
|
||||
opcache.DeleteUUID(jobUID)
|
||||
return c.SendString(elements.ErrorProgress(status.Error.Error(), status.GalleryElementName))
|
||||
}
|
||||
|
||||
return c.SendString(elements.ProgressBar(fmt.Sprint(status.Progress)))
|
||||
})
|
||||
|
||||
// Job completion route
|
||||
app.Get("/browse/backend/job/:uid", func(c *fiber.Ctx) error {
|
||||
jobUID := strings.Clone(c.Params("uid")) // note: strings.Clone is required for multiple requests!
|
||||
|
||||
status := galleryService.GetStatus(jobUID)
|
||||
|
||||
backendID := status.GalleryElementName
|
||||
opcache.DeleteUUID(jobUID)
|
||||
if backendID == "" {
|
||||
log.Debug().Msgf("no processing backend found for job: %+v\n", jobUID)
|
||||
}
|
||||
|
||||
log.Debug().Msgf("JOB finished: %+v\n", status)
|
||||
showDelete := true
|
||||
displayText := "Backend Installation completed"
|
||||
if status.Deletion {
|
||||
showDelete = false
|
||||
displayText = "Backend Deletion completed"
|
||||
}
|
||||
|
||||
return c.SendString(elements.DoneBackendProgress(backendID, displayText, showDelete))
|
||||
})
|
||||
}
|
||||
282
core/http/routes/ui_gallery.go
Normal file
282
core/http/routes/ui_gallery.go
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
package routes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/google/uuid"
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
"github.com/mudler/LocalAI/core/http/elements"
|
||||
"github.com/mudler/LocalAI/core/http/utils"
|
||||
"github.com/mudler/LocalAI/core/p2p"
|
||||
"github.com/mudler/LocalAI/core/services"
|
||||
"github.com/mudler/LocalAI/internal"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
func registerGalleryRoutes(app *fiber.App, cl *config.BackendConfigLoader, appConfig *config.ApplicationConfig, galleryService *services.GalleryService, opcache *services.OpCache) {
|
||||
|
||||
// Show the Models page (all models)
|
||||
app.Get("/browse", func(c *fiber.Ctx) error {
|
||||
term := c.Query("term")
|
||||
page := c.Query("page")
|
||||
items := c.Query("items")
|
||||
|
||||
models, err := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.ModelPath)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("could not list models from galleries")
|
||||
return c.Status(fiber.StatusInternalServerError).Render("views/error", fiber.Map{
|
||||
"Title": "LocalAI - Models",
|
||||
"BaseURL": utils.BaseURL(c),
|
||||
"Version": internal.PrintableVersion(),
|
||||
"ErrorCode": "500",
|
||||
"ErrorMessage": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Get all available tags
|
||||
allTags := map[string]struct{}{}
|
||||
tags := []string{}
|
||||
for _, m := range models {
|
||||
for _, t := range m.Tags {
|
||||
allTags[t] = struct{}{}
|
||||
}
|
||||
}
|
||||
for t := range allTags {
|
||||
tags = append(tags, t)
|
||||
}
|
||||
sort.Strings(tags)
|
||||
|
||||
if term != "" {
|
||||
models = gallery.GalleryElements[*gallery.GalleryModel](models).Search(term)
|
||||
}
|
||||
|
||||
// Get model statuses
|
||||
processingModelsData, taskTypes := opcache.GetStatus()
|
||||
|
||||
summary := fiber.Map{
|
||||
"Title": "LocalAI - Models",
|
||||
"BaseURL": utils.BaseURL(c),
|
||||
"Version": internal.PrintableVersion(),
|
||||
"Models": template.HTML(elements.ListModels(models, opcache, galleryService)),
|
||||
"Repositories": appConfig.Galleries,
|
||||
"AllTags": tags,
|
||||
"ProcessingModels": processingModelsData,
|
||||
"AvailableModels": len(models),
|
||||
"IsP2PEnabled": p2p.IsP2PEnabled(),
|
||||
|
||||
"TaskTypes": taskTypes,
|
||||
// "ApplicationConfig": appConfig,
|
||||
}
|
||||
|
||||
if page == "" {
|
||||
page = "1"
|
||||
}
|
||||
|
||||
if page != "" {
|
||||
// return a subset of the models
|
||||
pageNum, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
return c.Status(fiber.StatusBadRequest).SendString("Invalid page number")
|
||||
}
|
||||
|
||||
if pageNum == 0 {
|
||||
return c.Render("views/models", summary)
|
||||
}
|
||||
|
||||
itemsNum, err := strconv.Atoi(items)
|
||||
if err != nil {
|
||||
itemsNum = 21
|
||||
}
|
||||
|
||||
totalPages := int(math.Ceil(float64(len(models)) / float64(itemsNum)))
|
||||
|
||||
models = models.Paginate(pageNum, itemsNum)
|
||||
|
||||
prevPage := pageNum - 1
|
||||
nextPage := pageNum + 1
|
||||
if prevPage < 1 {
|
||||
prevPage = 1
|
||||
}
|
||||
if nextPage > totalPages {
|
||||
nextPage = totalPages
|
||||
}
|
||||
if prevPage != pageNum {
|
||||
summary["PrevPage"] = prevPage
|
||||
}
|
||||
summary["NextPage"] = nextPage
|
||||
summary["TotalPages"] = totalPages
|
||||
summary["CurrentPage"] = pageNum
|
||||
summary["Models"] = template.HTML(elements.ListModels(models, opcache, galleryService))
|
||||
}
|
||||
|
||||
// Render index
|
||||
return c.Render("views/models", summary)
|
||||
})
|
||||
|
||||
// Show the models, filtered from the user input
|
||||
// https://htmx.org/examples/active-search/
|
||||
app.Post("/browse/search/models", func(c *fiber.Ctx) error {
|
||||
page := c.Query("page")
|
||||
items := c.Query("items")
|
||||
|
||||
form := struct {
|
||||
Search string `form:"search"`
|
||||
}{}
|
||||
if err := c.BodyParser(&form); err != nil {
|
||||
return c.Status(fiber.StatusBadRequest).SendString(bluemonday.StrictPolicy().Sanitize(err.Error()))
|
||||
}
|
||||
|
||||
models, _ := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.ModelPath)
|
||||
|
||||
if page != "" {
|
||||
// return a subset of the models
|
||||
pageNum, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
return c.Status(fiber.StatusBadRequest).SendString("Invalid page number")
|
||||
}
|
||||
|
||||
itemsNum, err := strconv.Atoi(items)
|
||||
if err != nil {
|
||||
itemsNum = 21
|
||||
}
|
||||
|
||||
models = models.Paginate(pageNum, itemsNum)
|
||||
}
|
||||
|
||||
if form.Search != "" {
|
||||
models = models.Search(form.Search)
|
||||
}
|
||||
|
||||
return c.SendString(elements.ListModels(models, opcache, galleryService))
|
||||
})
|
||||
|
||||
/*
|
||||
|
||||
Install routes
|
||||
|
||||
*/
|
||||
|
||||
// This route is used when the "Install" button is pressed, we submit here a new job to the gallery service
|
||||
// https://htmx.org/examples/progress-bar/
|
||||
app.Post("/browse/install/model/:id", func(c *fiber.Ctx) error {
|
||||
galleryID := strings.Clone(c.Params("id")) // note: strings.Clone is required for multiple requests!
|
||||
log.Debug().Msgf("UI job submitted to install : %+v\n", galleryID)
|
||||
|
||||
id, err := uuid.NewUUID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uid := id.String()
|
||||
|
||||
opcache.Set(galleryID, uid)
|
||||
|
||||
op := services.GalleryOp[gallery.GalleryModel]{
|
||||
ID: uid,
|
||||
GalleryElementName: galleryID,
|
||||
Galleries: appConfig.Galleries,
|
||||
}
|
||||
go func() {
|
||||
galleryService.ModelGalleryChannel <- op
|
||||
}()
|
||||
|
||||
return c.SendString(elements.StartModelProgressBar(uid, "0", "Installation"))
|
||||
})
|
||||
|
||||
// This route is used when the "Install" button is pressed, we submit here a new job to the gallery service
|
||||
// https://htmx.org/examples/progress-bar/
|
||||
app.Post("/browse/delete/model/:id", func(c *fiber.Ctx) error {
|
||||
galleryID := strings.Clone(c.Params("id")) // note: strings.Clone is required for multiple requests!
|
||||
log.Debug().Msgf("UI job submitted to delete : %+v\n", galleryID)
|
||||
var galleryName = galleryID
|
||||
if strings.Contains(galleryID, "@") {
|
||||
// if the galleryID contains a @ it means that it's a model from a gallery
|
||||
// but we want to delete it from the local models which does not need
|
||||
// a repository ID
|
||||
galleryName = strings.Split(galleryID, "@")[1]
|
||||
}
|
||||
|
||||
id, err := uuid.NewUUID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uid := id.String()
|
||||
|
||||
// Track the deletion job by galleryID and galleryName
|
||||
// The GalleryID contains information about the repository,
|
||||
// while the GalleryName is ONLY the name of the model
|
||||
opcache.Set(galleryName, uid)
|
||||
opcache.Set(galleryID, uid)
|
||||
|
||||
op := services.GalleryOp[gallery.GalleryModel]{
|
||||
ID: uid,
|
||||
Delete: true,
|
||||
GalleryElementName: galleryName,
|
||||
Galleries: appConfig.Galleries,
|
||||
}
|
||||
go func() {
|
||||
galleryService.ModelGalleryChannel <- op
|
||||
cl.RemoveBackendConfig(galleryName)
|
||||
}()
|
||||
|
||||
return c.SendString(elements.StartModelProgressBar(uid, "0", "Deletion"))
|
||||
})
|
||||
|
||||
// Display the job current progress status
|
||||
// If the job is done, we trigger the /browse/job/:uid route
|
||||
// https://htmx.org/examples/progress-bar/
|
||||
app.Get("/browse/job/progress/:uid", func(c *fiber.Ctx) error {
|
||||
jobUID := strings.Clone(c.Params("uid")) // note: strings.Clone is required for multiple requests!
|
||||
|
||||
status := galleryService.GetStatus(jobUID)
|
||||
if status == nil {
|
||||
//fmt.Errorf("could not find any status for ID")
|
||||
return c.SendString(elements.ProgressBar("0"))
|
||||
}
|
||||
|
||||
if status.Progress == 100 {
|
||||
c.Set("HX-Trigger", "done") // this triggers /browse/job/:uid (which is when the job is done)
|
||||
return c.SendString(elements.ProgressBar("100"))
|
||||
}
|
||||
if status.Error != nil {
|
||||
// TODO: instead of deleting the job, we should keep it in the cache and make it dismissable by the user
|
||||
opcache.DeleteUUID(jobUID)
|
||||
return c.SendString(elements.ErrorProgress(status.Error.Error(), status.GalleryElementName))
|
||||
}
|
||||
|
||||
return c.SendString(elements.ProgressBar(fmt.Sprint(status.Progress)))
|
||||
})
|
||||
|
||||
// this route is hit when the job is done, and we display the
|
||||
// final state (for now just displays "Installation completed")
|
||||
app.Get("/browse/job/:uid", func(c *fiber.Ctx) error {
|
||||
jobUID := strings.Clone(c.Params("uid")) // note: strings.Clone is required for multiple requests!
|
||||
|
||||
status := galleryService.GetStatus(jobUID)
|
||||
|
||||
galleryID := status.GalleryElementName
|
||||
opcache.DeleteUUID(jobUID)
|
||||
if galleryID == "" {
|
||||
log.Debug().Msgf("no processing model found for job : %+v\n", jobUID)
|
||||
}
|
||||
|
||||
log.Debug().Msgf("JOB finished : %+v\n", status)
|
||||
showDelete := true
|
||||
displayText := "Installation completed"
|
||||
if status.Deletion {
|
||||
showDelete = false
|
||||
displayText = "Deletion completed"
|
||||
}
|
||||
|
||||
return c.SendString(elements.DoneModelProgress(galleryID, displayText, showDelete))
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue