feat(p2p): add network explorer and community pools (#3125)

* WIP Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Wire up a simple explorer DB Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * wip Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * WIP Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * refactor: group services id so can be identified easily in the ledger table Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(discovery): discovery service now gather worker informations correctly Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display network token Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display form to add new networks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): stop from overwriting networks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display only networks with active workers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): list only clusters in a network if it has online workers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * remove invalid and inactive networks if networks have no workers delete them from the database, similarly, if invalid. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: add workflow to deploy new explorer versions automatically Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * build-api: build with p2p tag Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Allow to specify a connection timeout Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * logging Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Better p2p defaults Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Set loglevel Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fix dht enable Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Default to info for loglevel Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add navbar Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Slightly improve rendering Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Allow to copy the token easily Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
2025-05-28 14:35:00 +00:00 · 2024-08-09 20:12:01 +02:00 · 2024-08-09 20:12:01 +02:00 · 9e3e892ac7
commit 9e3e892ac7
parent 5fcafc3d1e
19 changed files with 1082 additions and 17 deletions
--- a/core/http/endpoints/explorer/dashboard.go
+++ b/core/http/endpoints/explorer/dashboard.go
@ -0,0 +1,105 @@
+package explorer
+
+import (
+	"encoding/base64"
+	"sort"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/mudler/LocalAI/core/explorer"
+	"github.com/mudler/LocalAI/internal"
+)
+
+func Dashboard() func(*fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+
+		summary := fiber.Map{
+			"Title":   "LocalAI API - " + internal.PrintableVersion(),
+			"Version": internal.PrintableVersion(),
+		}
+
+		if string(c.Context().Request.Header.ContentType()) == "application/json" || len(c.Accepts("html")) == 0 {
+			// The client expects a JSON response
+			return c.Status(fiber.StatusOK).JSON(summary)
+		} else {
+			// Render index
+			return c.Render("views/explorer", summary)
+		}
+	}
+}
+
+type AddNetworkRequest struct {
+	Token       string `json:"token"`
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+type Network struct {
+	explorer.Network
+	explorer.TokenData
+	Token string `json:"token"`
+}
+
+func ShowNetworks(db *explorer.Database, ds *explorer.DiscoveryServer) func(*fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		networkState := ds.NetworkState()
+		results := []Network{}
+		for token, network := range networkState.Networks {
+			networkData, exists := db.Get(token) // get the token data
+			hasWorkers := false
+			for _, cluster := range network.Clusters {
+				if len(cluster.Workers) > 0 {
+					hasWorkers = true
+					break
+				}
+			}
+			if exists && hasWorkers {
+				results = append(results, Network{Network: network, TokenData: networkData, Token: token})
+			}
+		}
+
+		// order by number of clusters
+		sort.Slice(results, func(i, j int) bool {
+			return len(results[i].Clusters) > len(results[j].Clusters)
+		})
+
+		return c.JSON(results)
+	}
+}
+
+func AddNetwork(db *explorer.Database) func(*fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		request := new(AddNetworkRequest)
+		if err := c.BodyParser(request); err != nil {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Cannot parse JSON"})
+		}
+
+		if request.Token == "" {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Token is required"})
+		}
+
+		if request.Name == "" {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Name is required"})
+		}
+
+		if request.Description == "" {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Description is required"})
+		}
+
+		// TODO: check if token is valid, otherwise reject
+		// try to decode the token from base64
+		_, err := base64.StdEncoding.DecodeString(request.Token)
+		if err != nil {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Invalid token"})
+		}
+
+		if _, exists := db.Get(request.Token); exists {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Token already exists"})
+		}
+		err = db.Set(request.Token, explorer.TokenData{Name: request.Name, Description: request.Description})
+		if err != nil {
+			return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{"error": "Cannot add token"})
+		}
+
+		return c.Status(fiber.StatusOK).JSON(fiber.Map{"message": "Token added"})
+	}
+}