feat(p2p): add network explorer and community pools (#3125)

* WIP Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Wire up a simple explorer DB Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * wip Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * WIP Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * refactor: group services id so can be identified easily in the ledger table Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(discovery): discovery service now gather worker informations correctly Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display network token Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display form to add new networks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): stop from overwriting networks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display only networks with active workers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): list only clusters in a network if it has online workers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * remove invalid and inactive networks if networks have no workers delete them from the database, similarly, if invalid. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: add workflow to deploy new explorer versions automatically Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * build-api: build with p2p tag Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Allow to specify a connection timeout Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * logging Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Better p2p defaults Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Set loglevel Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fix dht enable Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Default to info for loglevel Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add navbar Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Slightly improve rendering Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Allow to copy the token easily Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
2025-05-20 10:35:01 +00:00 · 2024-08-09 20:12:01 +02:00 · 2024-08-09 20:12:01 +02:00 · 9e3e892ac7
commit 9e3e892ac7
parent 5fcafc3d1e
19 changed files with 1082 additions and 17 deletions
--- a/core/explorer/database.go
+++ b/core/explorer/database.go
@ -0,0 +1,106 @@
+package explorer
+
+// A simple JSON database for storing and retrieving p2p network tokens and a name and description.
+
+import (
+	"encoding/json"
+	"os"
+	"sort"
+	"sync"
+)
+
+// Database is a simple JSON database for storing and retrieving p2p network tokens and a name and description.
+type Database struct {
+	sync.RWMutex
+	path string
+	data map[string]TokenData
+}
+
+// TokenData is a p2p network token with a name and description.
+type TokenData struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+// NewDatabase creates a new Database with the given path.
+func NewDatabase(path string) (*Database, error) {
+	db := &Database{
+		data: make(map[string]TokenData),
+		path: path,
+	}
+	return db, db.load()
+}
+
+// Get retrieves a Token from the Database by its token.
+func (db *Database) Get(token string) (TokenData, bool) {
+	db.RLock()
+	defer db.RUnlock()
+	t, ok := db.data[token]
+	return t, ok
+}
+
+// Set stores a Token in the Database by its token.
+func (db *Database) Set(token string, t TokenData) error {
+	db.Lock()
+	db.data[token] = t
+	db.Unlock()
+
+	return db.Save()
+}
+
+// Delete removes a Token from the Database by its token.
+func (db *Database) Delete(token string) error {
+	db.Lock()
+	delete(db.data, token)
+	db.Unlock()
+	return db.Save()
+}
+
+func (db *Database) TokenList() []string {
+	db.RLock()
+	defer db.RUnlock()
+	tokens := []string{}
+	for k := range db.data {
+		tokens = append(tokens, k)
+	}
+
+	sort.Slice(tokens, func(i, j int) bool {
+		// sort by token
+		return tokens[i] < tokens[j]
+	})
+
+	return tokens
+}
+
+// load reads the Database from disk.
+func (db *Database) load() error {
+	db.Lock()
+	defer db.Unlock()
+
+	if _, err := os.Stat(db.path); os.IsNotExist(err) {
+		return nil
+	}
+
+	// Read the file from disk
+	// Unmarshal the JSON into db.data
+	f, err := os.ReadFile(db.path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(f, &db.data)
+}
+
+// Save writes the Database to disk.
+func (db *Database) Save() error {
+	db.RLock()
+	defer db.RUnlock()
+
+	// Marshal db.data into JSON
+	// Write the JSON to the file
+	f, err := os.Create(db.path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return json.NewEncoder(f).Encode(db.data)
+}