mirror of
https://github.com/mudler/LocalAI.git
synced 2025-05-28 06:25:00 +00:00
feat(p2p): add network explorer and community pools (#3125)
* WIP Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Wire up a simple explorer DB Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * wip Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * WIP Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * refactor: group services id so can be identified easily in the ledger table Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(discovery): discovery service now gather worker informations correctly Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display network token Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display form to add new networks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): stop from overwriting networks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display only networks with active workers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): list only clusters in a network if it has online workers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * remove invalid and inactive networks if networks have no workers delete them from the database, similarly, if invalid. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: add workflow to deploy new explorer versions automatically Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * build-api: build with p2p tag Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Allow to specify a connection timeout Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * logging Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Better p2p defaults Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Set loglevel Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fix dht enable Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Default to info for loglevel Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add navbar Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Slightly improve rendering Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Allow to copy the token easily Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
GitHub
parent
5fcafc3d1e
commit
9e3e892ac7
19 changed files with 1082 additions and 17 deletions
106
core/explorer/database.go
Normal file
106
core/explorer/database.go
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
package explorer
|
||||
|
||||
// A simple JSON database for storing and retrieving p2p network tokens and a name and description.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Database is a simple JSON database for storing and retrieving p2p network tokens and a name and description.
|
||||
type Database struct {
|
||||
sync.RWMutex
|
||||
path string
|
||||
data map[string]TokenData
|
||||
}
|
||||
|
||||
// TokenData is a p2p network token with a name and description.
|
||||
type TokenData struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
// NewDatabase creates a new Database with the given path.
|
||||
func NewDatabase(path string) (*Database, error) {
|
||||
db := &Database{
|
||||
data: make(map[string]TokenData),
|
||||
path: path,
|
||||
}
|
||||
return db, db.load()
|
||||
}
|
||||
|
||||
// Get retrieves a Token from the Database by its token.
|
||||
func (db *Database) Get(token string) (TokenData, bool) {
|
||||
db.RLock()
|
||||
defer db.RUnlock()
|
||||
t, ok := db.data[token]
|
||||
return t, ok
|
||||
}
|
||||
|
||||
// Set stores a Token in the Database by its token.
|
||||
func (db *Database) Set(token string, t TokenData) error {
|
||||
db.Lock()
|
||||
db.data[token] = t
|
||||
db.Unlock()
|
||||
|
||||
return db.Save()
|
||||
}
|
||||
|
||||
// Delete removes a Token from the Database by its token.
|
||||
func (db *Database) Delete(token string) error {
|
||||
db.Lock()
|
||||
delete(db.data, token)
|
||||
db.Unlock()
|
||||
return db.Save()
|
||||
}
|
||||
|
||||
func (db *Database) TokenList() []string {
|
||||
db.RLock()
|
||||
defer db.RUnlock()
|
||||
tokens := []string{}
|
||||
for k := range db.data {
|
||||
tokens = append(tokens, k)
|
||||
}
|
||||
|
||||
sort.Slice(tokens, func(i, j int) bool {
|
||||
// sort by token
|
||||
return tokens[i] < tokens[j]
|
||||
})
|
||||
|
||||
return tokens
|
||||
}
|
||||
|
||||
// load reads the Database from disk.
|
||||
func (db *Database) load() error {
|
||||
db.Lock()
|
||||
defer db.Unlock()
|
||||
|
||||
if _, err := os.Stat(db.path); os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read the file from disk
|
||||
// Unmarshal the JSON into db.data
|
||||
f, err := os.ReadFile(db.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(f, &db.data)
|
||||
}
|
||||
|
||||
// Save writes the Database to disk.
|
||||
func (db *Database) Save() error {
|
||||
db.RLock()
|
||||
defer db.RUnlock()
|
||||
|
||||
// Marshal db.data into JSON
|
||||
// Write the JSON to the file
|
||||
f, err := os.Create(db.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return json.NewEncoder(f).Encode(db.data)
|
||||
}
|
||||
92
core/explorer/database_test.go
Normal file
92
core/explorer/database_test.go
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
package explorer_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/explorer"
|
||||
)
|
||||
|
||||
var _ = Describe("Database", func() {
|
||||
var (
|
||||
dbPath string
|
||||
db *explorer.Database
|
||||
err error
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
// Create a temporary file path for the database
|
||||
dbPath = "test_db.json"
|
||||
db, err = explorer.NewDatabase(dbPath)
|
||||
Expect(err).To(BeNil())
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
// Clean up the temporary database file
|
||||
os.Remove(dbPath)
|
||||
})
|
||||
|
||||
Context("when managing tokens", func() {
|
||||
It("should add and retrieve a token", func() {
|
||||
token := "token123"
|
||||
t := explorer.TokenData{Name: "TokenName", Description: "A test token"}
|
||||
|
||||
err = db.Set(token, t)
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
retrievedToken, exists := db.Get(token)
|
||||
Expect(exists).To(BeTrue())
|
||||
Expect(retrievedToken).To(Equal(t))
|
||||
})
|
||||
|
||||
It("should delete a token", func() {
|
||||
token := "token123"
|
||||
t := explorer.TokenData{Name: "TokenName", Description: "A test token"}
|
||||
|
||||
err = db.Set(token, t)
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
err = db.Delete(token)
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
_, exists := db.Get(token)
|
||||
Expect(exists).To(BeFalse())
|
||||
})
|
||||
|
||||
It("should persist data to disk", func() {
|
||||
token := "token123"
|
||||
t := explorer.TokenData{Name: "TokenName", Description: "A test token"}
|
||||
|
||||
err = db.Set(token, t)
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
// Recreate the database object to simulate reloading from disk
|
||||
db, err = explorer.NewDatabase(dbPath)
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
retrievedToken, exists := db.Get(token)
|
||||
Expect(exists).To(BeTrue())
|
||||
Expect(retrievedToken).To(Equal(t))
|
||||
|
||||
// Check the token list
|
||||
tokenList := db.TokenList()
|
||||
Expect(tokenList).To(ContainElement(token))
|
||||
})
|
||||
})
|
||||
|
||||
Context("when loading an empty or non-existent file", func() {
|
||||
It("should start with an empty database", func() {
|
||||
dbPath = "empty_db.json"
|
||||
db, err = explorer.NewDatabase(dbPath)
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
_, exists := db.Get("nonexistent")
|
||||
Expect(exists).To(BeFalse())
|
||||
|
||||
// Clean up
|
||||
os.Remove(dbPath)
|
||||
})
|
||||
})
|
||||
})
|
||||
203
core/explorer/discovery.go
Normal file
203
core/explorer/discovery.go
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
package explorer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"github.com/mudler/LocalAI/core/p2p"
|
||||
"github.com/mudler/edgevpn/pkg/blockchain"
|
||||
)
|
||||
|
||||
type DiscoveryServer struct {
|
||||
sync.Mutex
|
||||
database *Database
|
||||
networkState *NetworkState
|
||||
connectionTime time.Duration
|
||||
}
|
||||
|
||||
type NetworkState struct {
|
||||
Networks map[string]Network
|
||||
}
|
||||
|
||||
func (s *DiscoveryServer) NetworkState() *NetworkState {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
return s.networkState
|
||||
}
|
||||
|
||||
// NewDiscoveryServer creates a new DiscoveryServer with the given Database.
|
||||
// it keeps the db state in sync with the network state
|
||||
func NewDiscoveryServer(db *Database, dur time.Duration) *DiscoveryServer {
|
||||
if dur == 0 {
|
||||
dur = 50 * time.Second
|
||||
}
|
||||
return &DiscoveryServer{
|
||||
database: db,
|
||||
connectionTime: dur,
|
||||
networkState: &NetworkState{
|
||||
Networks: map[string]Network{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type Network struct {
|
||||
Clusters []ClusterData
|
||||
}
|
||||
|
||||
func (s *DiscoveryServer) runBackground() {
|
||||
if len(s.database.TokenList()) == 0 {
|
||||
time.Sleep(5 * time.Second) // avoid busy loop
|
||||
return
|
||||
}
|
||||
|
||||
for _, token := range s.database.TokenList() {
|
||||
c, cancel := context.WithTimeout(context.Background(), s.connectionTime)
|
||||
defer cancel()
|
||||
|
||||
// Connect to the network
|
||||
// Get the number of nodes
|
||||
// save it in the current state (mutex)
|
||||
// do not do in parallel
|
||||
n, err := p2p.NewNode(token)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("Failed to create node")
|
||||
s.database.Delete(token)
|
||||
continue
|
||||
}
|
||||
|
||||
err = n.Start(c)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("Failed to start node")
|
||||
s.database.Delete(token)
|
||||
continue
|
||||
}
|
||||
|
||||
ledger, err := n.Ledger()
|
||||
if err != nil {
|
||||
log.Err(err).Msg("Failed to start ledger")
|
||||
s.database.Delete(token)
|
||||
continue
|
||||
}
|
||||
|
||||
networkData := make(chan ClusterData)
|
||||
|
||||
// get the network data - it takes the whole timeout
|
||||
// as we might not be connected to the network yet,
|
||||
// and few attempts would have to be made before bailing out
|
||||
go s.retrieveNetworkData(c, ledger, networkData)
|
||||
|
||||
hasWorkers := false
|
||||
ledgerK := []ClusterData{}
|
||||
for key := range networkData {
|
||||
ledgerK = append(ledgerK, key)
|
||||
if len(key.Workers) > 0 {
|
||||
hasWorkers = true
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug().Any("network", token).Msgf("Network has %d clusters", len(ledgerK))
|
||||
if len(ledgerK) != 0 {
|
||||
for _, k := range ledgerK {
|
||||
log.Debug().Any("network", token).Msgf("Clusterdata %+v", k)
|
||||
}
|
||||
}
|
||||
|
||||
if hasWorkers {
|
||||
s.Lock()
|
||||
s.networkState.Networks[token] = Network{
|
||||
Clusters: ledgerK,
|
||||
}
|
||||
s.Unlock()
|
||||
} else {
|
||||
log.Info().Any("network", token).Msg("No workers found in the network. Removing it from the database")
|
||||
s.database.Delete(token)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type ClusterData struct {
|
||||
Workers []string
|
||||
Type string
|
||||
NetworkID string
|
||||
}
|
||||
|
||||
func (s *DiscoveryServer) retrieveNetworkData(c context.Context, ledger *blockchain.Ledger, networkData chan ClusterData) {
|
||||
clusters := map[string]ClusterData{}
|
||||
|
||||
defer func() {
|
||||
for _, n := range clusters {
|
||||
networkData <- n
|
||||
}
|
||||
close(networkData)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-c.Done():
|
||||
return
|
||||
default:
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
data := ledger.LastBlock().Storage
|
||||
LEDGER:
|
||||
for d := range data {
|
||||
toScanForWorkers := false
|
||||
cd := ClusterData{}
|
||||
isWorkerCluster := d == p2p.WorkerID || (strings.Contains(d, "_") && strings.Contains(d, p2p.WorkerID))
|
||||
isFederatedCluster := d == p2p.FederatedID || (strings.Contains(d, "_") && strings.Contains(d, p2p.FederatedID))
|
||||
switch {
|
||||
case isWorkerCluster:
|
||||
toScanForWorkers = true
|
||||
cd.Type = "worker"
|
||||
case isFederatedCluster:
|
||||
toScanForWorkers = true
|
||||
cd.Type = "federated"
|
||||
}
|
||||
|
||||
if strings.Contains(d, "_") {
|
||||
cd.NetworkID = strings.Split(d, "_")[0]
|
||||
}
|
||||
|
||||
if !toScanForWorkers {
|
||||
continue LEDGER
|
||||
}
|
||||
|
||||
atLeastOneWorker := false
|
||||
DATA:
|
||||
for _, v := range data[d] {
|
||||
nd := &p2p.NodeData{}
|
||||
if err := v.Unmarshal(nd); err != nil {
|
||||
continue DATA
|
||||
}
|
||||
|
||||
if nd.IsOnline() {
|
||||
atLeastOneWorker = true
|
||||
(&cd).Workers = append(cd.Workers, nd.ID)
|
||||
}
|
||||
}
|
||||
|
||||
if atLeastOneWorker {
|
||||
clusters[d] = cd
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start the discovery server. This is meant to be run in to a goroutine.
|
||||
func (s *DiscoveryServer) Start(ctx context.Context) error {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("context cancelled")
|
||||
default:
|
||||
// Collect data
|
||||
s.runBackground()
|
||||
}
|
||||
}
|
||||
}
|
||||
13
core/explorer/explorer_suite_test.go
Normal file
13
core/explorer/explorer_suite_test.go
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
package explorer_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestExplorer(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Explorer test suite")
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue