feat: Add backend gallery (#5607)

* feat: Add backend gallery This PR add support to manage backends as similar to models. There is now available a backend gallery which can be used to install and remove extra backends. The backend gallery can be configured similarly as a model gallery, and API calls allows to install and remove new backends in runtime, and as well during the startup phase of LocalAI. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add backends docs Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * wip: Backend Dockerfile for python backends Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat: drop extras images, build python backends separately Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixup on all backends Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * test CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Tweaks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Drop old backends leftovers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixup CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Move dockerfile upper Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fix proto Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Feature dropped for consistency - we prefer model galleries Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add missing packages in the build image Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * exllama is ponly available on cublas Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * pin torch on chatterbox Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixups to index Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Debug CI * Install accellerators deps Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add target arch * Add cuda minor version Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Use self-hosted runners Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: use quay for test images Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixups for vllm and chatterbox Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Small fixups on CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chatterbox is only available for nvidia Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Simplify CI builds Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Adapt test, use qwen3 Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chore(model gallery): add jina-reranker-v1-tiny-en-gguf Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(gguf-parser): recover from potential panics that can happen while reading ggufs with gguf-parser Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Use reranker from llama.cpp in AIO images Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Limit concurrent jobs Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>
2025-06-17 08:15:00 +00:00 · 2025-06-15 14:56:52 +02:00 · 2025-06-15 14:56:52 +02:00 · 2d64269763
commit 2d64269763
parent a7a6020328
114 changed files with 3996 additions and 1382 deletions
--- a/core/http/elements/gallery.go
+++ b/core/http/elements/gallery.go
@ -331,7 +331,7 @@ func modelActionItems(m *gallery.GalleryModel, processTracker ProcessTracker, ga
 				elem.If(
 					currentlyProcessing,
 					elem.Node( // If currently installing, show progress bar
-						elem.Raw(StartProgressBar(jobID, "0", progressMessage)),
+						elem.Raw(StartModelProgressBar(jobID, "0", progressMessage)),
 					), // Otherwise, show install button (if not installed) or display "Installed"
 					elem.If(m.Installed,
 						elem.Node(elem.Div(
@ -418,3 +418,335 @@ func ListModels(models []*gallery.GalleryModel, processTracker ProcessTracker, g

 	return wrapper.Render()
 }
+
+func ListBackends(backends []*gallery.GalleryBackend, processTracker ProcessTracker, galleryService *services.GalleryService) string {
+	backendsElements := []elem.Node{}
+
+	for _, b := range backends {
+		elems := []elem.Node{}
+
+		if b.Icon == "" {
+			b.Icon = noImage
+		}
+
+		divProperties := attrs.Props{
+			"class": "flex justify-center items-center",
+		}
+
+		elems = append(elems,
+			elem.Div(divProperties,
+				elem.A(attrs.Props{
+					"href": "#!",
+				},
+					elem.Img(attrs.Props{
+						"class":   "rounded-t-lg max-h-48 max-w-96 object-cover mt-3",
+						"src":     b.Icon,
+						"loading": "lazy",
+					}),
+				),
+			),
+		)
+
+		elems = append(elems,
+			backendDescription(b),
+			backendActionItems(b, processTracker, galleryService),
+		)
+		backendsElements = append(backendsElements,
+			elem.Div(
+				attrs.Props{
+					"class": "me-4 mb-2 block rounded-lg bg-white shadow-secondary-1 dark:bg-gray-800 dark:bg-surface-dark dark:text-white text-surface pb-2 bg-gray-800/90 border border-gray-700/50 rounded-xl overflow-hidden transition-all duration-300 hover:shadow-lg hover:shadow-blue-900/20 hover:-translate-y-1 hover:border-blue-700/50",
+				},
+				elem.Div(
+					attrs.Props{},
+					elems...,
+				),
+			),
+			backendModal(b),
+		)
+	}
+
+	wrapper := elem.Div(attrs.Props{
+		"class": "dark grid grid-cols-1 grid-rows-1 md:grid-cols-3 block rounded-lg shadow-secondary-1 dark:bg-surface-dark",
+	}, backendsElements...)
+
+	return wrapper.Render()
+}
+
+func backendDescription(b *gallery.GalleryBackend) elem.Node {
+	return elem.Div(
+		attrs.Props{
+			"class": "p-6 text-surface dark:text-white",
+		},
+		elem.H5(
+			attrs.Props{
+				"class": "mb-2 text-xl font-bold leading-tight",
+			},
+			elem.Text(bluemonday.StrictPolicy().Sanitize(b.Name)),
+		),
+		elem.Div(
+			attrs.Props{
+				"class": "mb-4 text-sm truncate text-base",
+			},
+			elem.Text(bluemonday.StrictPolicy().Sanitize(b.Description)),
+		),
+	)
+}
+
+func backendActionItems(b *gallery.GalleryBackend, processTracker ProcessTracker, galleryService *services.GalleryService) elem.Node {
+	galleryID := fmt.Sprintf("%s@%s", b.Gallery.Name, b.Name)
+	currentlyProcessing := processTracker.Exists(galleryID)
+	jobID := ""
+	isDeletionOp := false
+	if currentlyProcessing {
+		status := galleryService.GetStatus(galleryID)
+		if status != nil && status.Deletion {
+			isDeletionOp = true
+		}
+		jobID = processTracker.Get(galleryID)
+	}
+
+	nodes := []elem.Node{
+		cardSpan("Repository: "+b.Gallery.Name, "fa-brands fa-git-alt"),
+	}
+
+	if b.License != "" {
+		nodes = append(nodes,
+			cardSpan("License: "+b.License, "fas fa-book"),
+		)
+	}
+
+	progressMessage := "Installation"
+	if isDeletionOp {
+		progressMessage = "Deletion"
+	}
+
+	return elem.Div(
+		attrs.Props{
+			"class": "px-6 pt-4 pb-2",
+		},
+		elem.P(
+			attrs.Props{
+				"class": "mb-4 text-base",
+			},
+			nodes...,
+		),
+		elem.Div(
+			attrs.Props{
+				"id":    "action-div-" + dropBadChars(galleryID),
+				"class": "flow-root",
+			},
+			backendInfoButton(b),
+			elem.Div(
+				attrs.Props{
+					"class": "float-right",
+				},
+				elem.If(
+					currentlyProcessing,
+					elem.Node(
+						elem.Raw(StartModelProgressBar(jobID, "0", progressMessage)),
+					),
+					elem.If(b.Installed,
+						elem.Node(elem.Div(
+							attrs.Props{},
+							backendReInstallButton(galleryID),
+							backendDeleteButton(galleryID),
+						)),
+						backendInstallButton(galleryID),
+					),
+				),
+			),
+		),
+	)
+}
+
+func backendModal(b *gallery.GalleryBackend) elem.Node {
+	urls := []elem.Node{}
+	for _, url := range b.URLs {
+		urls = append(urls,
+			elem.Li(attrs.Props{}, link(url, url)),
+		)
+	}
+
+	tagsNodes := []elem.Node{}
+	for _, tag := range b.Tags {
+		tagsNodes = append(tagsNodes,
+			searchableElement(tag, "fas fa-tag"),
+		)
+	}
+
+	modalID := fmt.Sprintf("modal-%s", dropBadChars(fmt.Sprintf("%s@%s", b.Gallery.Name, b.Name)))
+
+	return elem.Div(
+		attrs.Props{
+			"id":          modalID,
+			"tabindex":    "-1",
+			"aria-hidden": "true",
+			"class":       "hidden fixed top-0 right-0 left-0 z-50 justify-center items-center w-full md:inset-0 h-full max-h-full bg-gray-900/50",
+		},
+		elem.Div(
+			attrs.Props{
+				"class": "relative p-4 w-full max-w-2xl h-[90vh] mx-auto mt-[5vh]",
+			},
+			elem.Div(
+				attrs.Props{
+					"class": "relative bg-white rounded-lg shadow dark:bg-gray-700 h-full flex flex-col",
+				},
+				elem.Div(
+					attrs.Props{
+						"class": "flex items-center justify-between p-4 md:p-5 border-b rounded-t dark:border-gray-600",
+					},
+					elem.H3(
+						attrs.Props{
+							"class": "text-xl font-semibold text-gray-900 dark:text-white",
+						},
+						elem.Text(bluemonday.StrictPolicy().Sanitize(b.Name)),
+					),
+					elem.Button(
+						attrs.Props{
+							"class":           "text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm w-8 h-8 ms-auto inline-flex justify-center items-center dark:hover:bg-gray-600 dark:hover:text-white",
+							"data-modal-hide": modalID,
+						},
+						elem.Raw(
+							`<svg class="w-3 h-3" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 14 14">
+							<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="m1 1 6 6m0 0 6 6M7 7l6-6M7 7l-6 6"/>
+						</svg>`,
+						),
+						elem.Span(
+							attrs.Props{
+								"class": "sr-only",
+							},
+							elem.Text("Close modal"),
+						),
+					),
+				),
+				elem.Div(
+					attrs.Props{
+						"class": "p-4 md:p-5 space-y-4 overflow-y-auto flex-grow",
+					},
+					elem.Div(
+						attrs.Props{
+							"class": "flex justify-center items-center",
+						},
+						elem.Img(attrs.Props{
+							"src":     b.Icon,
+							"class":   "rounded-t-lg max-h-48 max-w-96 object-cover mt-3",
+							"loading": "lazy",
+						}),
+					),
+					elem.P(
+						attrs.Props{
+							"class": "text-base leading-relaxed text-gray-500 dark:text-gray-400",
+						},
+						elem.Text(bluemonday.StrictPolicy().Sanitize(b.Description)),
+					),
+					elem.Div(
+						attrs.Props{
+							"class": "flex flex-wrap gap-2",
+						},
+						tagsNodes...,
+					),
+					elem.Div(
+						attrs.Props{
+							"class": "text-base leading-relaxed text-gray-500 dark:text-gray-400",
+						},
+						elem.Ul(attrs.Props{}, urls...),
+					),
+				),
+				elem.Div(
+					attrs.Props{
+						"class": "flex items-center p-4 md:p-5 border-t border-gray-200 rounded-b dark:border-gray-600",
+					},
+					elem.Button(
+						attrs.Props{
+							"data-modal-hide": modalID,
+							"type":            "button",
+							"class":           "text-white bg-blue-700 hover:bg-blue-800 focus:ring-4 focus:outline-none focus:ring-blue-300 font-medium rounded-lg text-sm px-5 py-2.5 text-center dark:bg-blue-600 dark:hover:bg-blue-700 dark:focus:ring-blue-800",
+						},
+						elem.Text("Close"),
+					),
+				),
+			),
+		),
+	)
+}
+
+func backendInfoButton(b *gallery.GalleryBackend) elem.Node {
+	modalID := fmt.Sprintf("modal-%s", dropBadChars(fmt.Sprintf("%s@%s", b.Gallery.Name, b.Name)))
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"class":                 "inline-flex items-center rounded-lg bg-gray-700 hover:bg-gray-600 px-4 py-2 text-sm font-medium text-white transition duration-300 ease-in-out",
+			"data-modal-target":     modalID,
+			"data-modal-toggle":     modalID,
+		},
+		elem.P(
+			attrs.Props{
+				"class": "flex items-center",
+			},
+			elem.I(
+				attrs.Props{
+					"class": "fas fa-info-circle pr-2",
+				},
+			),
+			elem.Text("Info"),
+		),
+	)
+}
+
+func backendInstallButton(galleryID string) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"class":                 "float-right inline-flex items-center rounded-lg bg-blue-600 hover:bg-blue-700 px-4 py-2 text-sm font-medium text-white transition duration-300 ease-in-out shadow hover:shadow-lg",
+			"hx-swap":               "outerHTML",
+			"hx-post":               "browse/install/backend/" + galleryID,
+		},
+		elem.I(
+			attrs.Props{
+				"class": "fa-solid fa-download pr-2",
+			},
+		),
+		elem.Text("Install"),
+	)
+}
+
+func backendReInstallButton(galleryID string) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"class":                 "float-right inline-block rounded bg-primary ml-2 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-primary-accent-300 hover:shadow-primary-2 focus:bg-primary-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-primary-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
+			"hx-target":             "#action-div-" + dropBadChars(galleryID),
+			"hx-swap":               "outerHTML",
+			"hx-post":               "browse/install/backend/" + galleryID,
+		},
+		elem.I(
+			attrs.Props{
+				"class": "fa-solid fa-arrow-rotate-right pr-2",
+			},
+		),
+		elem.Text("Reinstall"),
+	)
+}
+
+func backendDeleteButton(galleryID string) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"hx-confirm":            "Are you sure you wish to delete the backend?",
+			"class":                 "float-right inline-block rounded bg-red-800 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-red-accent-300 hover:shadow-red-2 focus:bg-red-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-red-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
+			"hx-target":             "#action-div-" + dropBadChars(galleryID),
+			"hx-swap":               "outerHTML",
+			"hx-post":               "browse/delete/backend/" + galleryID,
+		},
+		elem.I(
+			attrs.Props{
+				"class": "fa-solid fa-cancel pr-2",
+			},
+		),
+		elem.Text("Delete"),
+	)
+}
--- a/core/http/elements/progressbar.go
+++ b/core/http/elements/progressbar.go
@ -6,7 +6,7 @@ import (
 	"github.com/microcosm-cc/bluemonday"
 )

-func DoneProgress(galleryID, text string, showDelete bool) string {
+func DoneModelProgress(galleryID, text string, showDelete bool) string {
 	return elem.Div(
 		attrs.Props{
 			"id": "action-div-" + dropBadChars(galleryID),
@ -24,6 +24,24 @@ func DoneProgress(galleryID, text string, showDelete bool) string {
 	).Render()
 }

+func DoneBackendProgress(galleryID, text string, showDelete bool) string {
+	return elem.Div(
+		attrs.Props{
+			"id": "action-div-" + dropBadChars(galleryID),
+		},
+		elem.H3(
+			attrs.Props{
+				"role":      "status",
+				"id":        "pblabel",
+				"tabindex":  "-1",
+				"autofocus": "",
+			},
+			elem.Text(bluemonday.StrictPolicy().Sanitize(text)),
+		),
+		elem.If(showDelete, backendDeleteButton(galleryID), reInstallButton(galleryID)),
+	).Render()
+}
+
 func ErrorProgress(err, galleryName string) string {
 	return elem.Div(
 		attrs.Props{},
@ -57,14 +75,22 @@ func ProgressBar(progress string) string {
 	).Render()
 }

-func StartProgressBar(uid, progress, text string) string {
+func StartModelProgressBar(uid, progress, text string) string {
+	return progressBar(uid, "browse/job/", progress, text)
+}
+
+func StartBackendProgressBar(uid, progress, text string) string {
+	return progressBar(uid, "browse/backend/job/", progress, text)
+}
+
+func progressBar(uid, url, progress, text string) string {
 	if progress == "" {
 		progress = "0"
 	}
 	return elem.Div(
 		attrs.Props{
 			"hx-trigger": "done",
-			"hx-get":     "browse/job/" + uid,
+			"hx-get":     url + uid,
 			"hx-swap":    "outerHTML",
 			"hx-target":  "this",
 		},
@ -77,7 +103,7 @@ func StartProgressBar(uid, progress, text string) string {
 			},
 			elem.Text(bluemonday.StrictPolicy().Sanitize(text)), //Perhaps overly defensive
 			elem.Div(attrs.Props{
-				"hx-get":     "browse/job/progress/" + uid,
+				"hx-get":     url + "progress/" + uid,
 				"hx-trigger": "every 600ms",
 				"hx-target":  "this",
 				"hx-swap":    "innerHTML",