feat: embedded model configurations, add popular model examples, refactoring (#1532)

* move downloader out

* separate startup functions for preloading configuration files

* docs: add popular model examples

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* shorteners

* Add llava

* Add mistral-openorca

* Better link to build section

* docs: update

* fixup

* Drop code dups

* Minor fixups

* Apply suggestions from code review

Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>

* ci: try to cache gRPC build during tests

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* ci: do not build all images for tests, just necessary

* ci: cache gRPC also in release pipeline

* fixes

* Update model_preload_test.go

Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>

embedded/embedded.go

@@ -0,0 +1,53 @@
+package embedded
+
+import (
+	"embed"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/go-skynet/LocalAI/pkg/assets"
+	"gopkg.in/yaml.v3"
+)
+
+var modelShorteners map[string]string
+
+//go:embed model_library.yaml
+var modelLibrary []byte
+
+//go:embed models/*
+var embeddedModels embed.FS
+
+func ModelShortURL(s string) string {
+	if _, ok := modelShorteners[s]; ok {
+		s = modelShorteners[s]
+	}
+
+	return s
+}
+
+func init() {
+	yaml.Unmarshal(modelLibrary, &modelShorteners)
+}
+
+// ExistsInModelsLibrary checks if a model exists in the embedded models library
+func ExistsInModelsLibrary(s string) bool {
+	f := fmt.Sprintf("%s.yaml", s)
+
+	a := []string{}
+
+	for _, j := range assets.ListFiles(embeddedModels) {
+		a = append(a, strings.TrimPrefix(j, "models/"))
+	}
+
+	return slices.Contains(a, f)
+}
+
+// ResolveContent returns the content in the embedded model library
+func ResolveContent(s string) ([]byte, error) {
+	if ExistsInModelsLibrary(s) {
+		return embeddedModels.ReadFile(fmt.Sprintf("models/%s.yaml", s))
+	}
+
+	return nil, fmt.Errorf("cannot find model %s", s)
+}