diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c0e1c051..1f20b889 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -96,6 +96,7 @@ jobs: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + go install github.com/GeertJohan/go.rice/rice@latest # The python3-grpc-tools package in 22.04 is too old pip install --user grpcio-tools @@ -183,6 +184,7 @@ jobs: rm protoc.zip go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + go install github.com/GeertJohan/go.rice/rice@latest PATH="$PATH:$HOME/go/bin" make protogen-go - name: Build images run: | @@ -222,6 +224,7 @@ jobs: run: | brew install protobuf grpc make protoc-gen-go protoc-gen-go-grpc libomp llvm pip install --user --no-cache-dir grpcio-tools + go install github.com/GeertJohan/go.rice/rice@latest - name: Test run: | export C_INCLUDE_PATH=/usr/local/include diff --git a/Dockerfile b/Dockerfile index 796a0d69..abbfc7a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,9 +46,10 @@ EOT RUN curl -L -s https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz | tar -C /usr/local -xz ENV PATH=$PATH:/root/go/bin:/usr/local/go/bin -# Install grpc compilers +# Install grpc compilers and rice RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 && \ - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af && \ + go install github.com/GeertJohan/go.rice/rice@latest COPY --chmod=644 custom-ca-certs/* /usr/local/share/ca-certificates/ RUN update-ca-certificates @@ -300,14 +301,7 @@ COPY .git . RUN make prepare ## Build the binary -## If it's CUDA or hipblas, we want to skip some of the llama-compat backends to save space -## We only leave the most CPU-optimized variant and the fallback for the cublas/hipblas build -## (both will use CUDA or hipblas for the actual computation) -RUN if [ "${BUILD_TYPE}" = "cublas" ] || [ "${BUILD_TYPE}" = "hipblas" ]; then \ - SKIP_GRPC_BACKEND="backend-assets/grpc/llama-cpp-avx512 backend-assets/grpc/llama-cpp-avx backend-assets/grpc/llama-cpp-avx2" make build; \ - else \ - make build; \ - fi +RUN make build RUN if [ ! -d "/build/sources/go-piper/piper-phonemize/pi/lib/" ]; then \ mkdir -p /build/sources/go-piper/piper-phonemize/pi/lib/ \ diff --git a/Makefile b/Makefile index 008e0bdf..fbdfe285 100644 --- a/Makefile +++ b/Makefile @@ -349,6 +349,7 @@ ifneq ($(BACKEND_LIBS),) cp -f $(BACKEND_LIBS) backend-assets/lib/ endif CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o $(BINARY_NAME) ./ + rice append --exec $(BINARY_NAME) build-minimal: BUILD_GRPC_FOR_BACKEND_LLAMA=true GRPC_BACKENDS="backend-assets/grpc/llama-cpp-avx2" GO_TAGS=p2p $(MAKE) build @@ -420,15 +421,20 @@ prepare-test: grpcs cp -rf backend-assets core/http cp tests/models_fixtures/* test-models -test: prepare test-models/testmodel.ggml grpcs - @echo 'Running tests' +## Build test binary +test-binary: prepare-test export GO_TAGS="tts debug" - $(MAKE) prepare-test + CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOCMD) test -c -o test-binary ./... + rice append --exec test-binary + +## Test targets +test: prepare test-models/testmodel.ggml test-binary HUGGINGFACE_GRPC=$(abspath ./)/backend/python/transformers/run.sh TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="!llama-gguf" --flake-attempts $(TEST_FLAKES) --fail-fast -v -r $(TEST_PATHS) + ./test-binary --label-filter="!llama-gguf" --flake-attempts $(TEST_FLAKES) --fail-fast -v -r $(TEST_PATHS) $(MAKE) test-llama-gguf $(MAKE) test-tts $(MAKE) test-stablediffusion + rm -f test-binary prepare-e2e: mkdir -p $(TEST_DIR) @@ -454,22 +460,26 @@ teardown-e2e: rm -rf $(TEST_DIR) || true docker stop $$(docker ps -q --filter ancestor=localai-tests) -test-llama-gguf: prepare-test +test-llama-gguf: test-binary TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="llama-gguf" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) + ./test-binary --label-filter="llama-gguf" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) + rm -f test-binary -test-tts: prepare-test +test-tts: test-binary TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="tts" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) + ./test-binary --label-filter="tts" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) + rm -f test-binary -test-stablediffusion: prepare-test +test-stablediffusion: test-binary TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="stablediffusion" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) + ./test-binary --label-filter="stablediffusion" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) + rm -f test-binary -test-stores: backend-assets/grpc/local-store +test-stores: backend-assets/grpc/local-store test-binary mkdir -p tests/integration/backend-assets/grpc cp -f backend-assets/grpc/local-store tests/integration/backend-assets/grpc/ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="stores" --flake-attempts $(TEST_FLAKES) -v -r tests/integration + ./test-binary --label-filter="stores" --flake-attempts $(TEST_FLAKES) -v -r tests/integration + rm -f test-binary test-container: docker build --target requirements -t local-ai-test-container . diff --git a/assets.go b/assets.go index 1acff154..b3c81387 100644 --- a/assets.go +++ b/assets.go @@ -1,6 +1,15 @@ package main -import "embed" +import ( + rice "github.com/GeertJohan/go.rice" +) -//go:embed backend-assets/* -var backendAssets embed.FS +var backendAssets *rice.Box + +func init() { + var err error + backendAssets, err = rice.FindBox("backend-assets") + if err != nil { + panic(err) + } +} diff --git a/core/cli/context/context.go b/core/cli/context/context.go index fa93408e..34242e97 100644 --- a/core/cli/context/context.go +++ b/core/cli/context/context.go @@ -1,11 +1,13 @@ package cliContext -import "embed" +import ( + rice "github.com/GeertJohan/go.rice" +) type Context struct { Debug bool `env:"LOCALAI_DEBUG,DEBUG" default:"false" hidden:"" help:"DEPRECATED, use --log-level=debug instead. Enable debug logging"` LogLevel *string `env:"LOCALAI_LOG_LEVEL" enum:"error,warn,info,debug,trace" help:"Set the level of logs to output [${enum}]"` // This field is not a command line argument/flag, the struct tag excludes it from the parsed CLI - BackendAssets embed.FS `kong:"-"` + BackendAssets *rice.Box `kong:"-"` } diff --git a/core/config/application_config.go b/core/config/application_config.go index 9648e454..81c00999 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -2,11 +2,11 @@ package config import ( "context" - "embed" "encoding/json" "regexp" "time" + rice "github.com/GeertJohan/go.rice" "github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/rs/zerolog/log" ) @@ -47,7 +47,7 @@ type ApplicationConfig struct { Galleries []Gallery - BackendAssets embed.FS + BackendAssets *rice.Box AssetsDestination string ExternalGRPCBackends map[string]string @@ -198,7 +198,7 @@ func WithBackendAssetsOutput(out string) AppOption { } } -func WithBackendAssets(f embed.FS) AppOption { +func WithBackendAssets(f *rice.Box) AppOption { return func(o *ApplicationConfig) { o.BackendAssets = f } diff --git a/core/http/app_test.go b/core/http/app_test.go index 8d12c496..2d243322 100644 --- a/core/http/app_test.go +++ b/core/http/app_test.go @@ -3,7 +3,6 @@ package http_test import ( "bytes" "context" - "embed" "encoding/json" "fmt" "io" @@ -24,6 +23,7 @@ import ( . "github.com/onsi/gomega" "gopkg.in/yaml.v3" + rice "github.com/GeertJohan/go.rice" openaigo "github.com/otiai10/openaigo" "github.com/sashabaranov/go-openai" "github.com/sashabaranov/go-openai/jsonschema" @@ -264,8 +264,15 @@ func getRequest(url string, header http.Header) (error, int, []byte) { const bertEmbeddingsURL = `https://gist.githubusercontent.com/mudler/0a080b166b87640e8644b09c2aee6e3b/raw/f0e8c26bb72edc16d9fbafbfd6638072126ff225/bert-embeddings-gallery.yaml` -//go:embed backend-assets/* -var backendAssets embed.FS +var backendAssets *rice.Box + +func init() { + var err error + backendAssets, err = rice.FindBox("backend-assets") + if err != nil { + panic(err) + } +} var _ = Describe("API test", func() { diff --git a/go.mod b/go.mod index 757376ab..7652cefc 100644 --- a/go.mod +++ b/go.mod @@ -75,6 +75,8 @@ require ( cloud.google.com/go/auth v0.4.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect + github.com/GeertJohan/go.rice v1.0.3 // indirect + github.com/daaku/go.zipexe v1.0.2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect github.com/fasthttp/websocket v1.5.3 // indirect diff --git a/go.sum b/go.sum index aad5d177..776f8e1c 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,9 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= +github.com/GeertJohan/go.rice v1.0.3 h1:k5viR+xGtIhF61125vCE1cmJ5957RQGXG6dmbaWZSmI= +github.com/GeertJohan/go.rice v1.0.3/go.mod h1:XVdrU4pW00M4ikZed5q56tPf1v2KwnIKeIdc9CBYNt4= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -39,6 +42,7 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEV github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU= github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.8.0 h1:w9WJUjFFmHHB2e8mRpL9jjy3alYDlU0QLDezj1xE264= @@ -108,6 +112,8 @@ github.com/creachadair/otp v0.5.0 h1:q3Th7CXm2zlmCdBjw5tEPFOj4oWJMnVL5HXlq0sNKS0 github.com/creachadair/otp v0.5.0/go.mod h1:0kceI87EnYFNYSTL121goJVAnk3eJhaed9H0nMuJUkA= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/daaku/go.zipexe v1.0.2 h1:Zg55YLYTr7M9wjKn8SY/WcpuuEi+kR2u4E8RhvpyXmk= +github.com/daaku/go.zipexe v1.0.2/go.mod h1:5xWogtqlYnfBXkSB1o9xysukNP9GTvaNkqzUZbt3Bw8= github.com/dave-gray101/v2keyauth v0.0.0-20240624150259-c45d584d25e2 h1:flLYmnQFZNo04x2NPehMbf30m7Pli57xwZ0NFqR/hb0= github.com/dave-gray101/v2keyauth v0.0.0-20240624150259-c45d584d25e2/go.mod h1:NtWqRzAp/1tw+twkW8uuBenEVVYndEAZACWU3F3xdoQ= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -531,6 +537,7 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nikolalohinski/gonja/v2 v2.3.2 h1:UgLFfqi7L9XfX0PEcE4eUpvGojVQL5KhBfJJaBp7ZxY= github.com/nikolalohinski/gonja/v2 v2.3.2/go.mod h1:1Wcc/5huTu6y36e0sOFR1XQoFlylw3c3H3L5WOz0RDg= +github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= @@ -772,6 +779,7 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8= github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= diff --git a/pkg/assets/extract.go b/pkg/assets/extract.go index e2e912f5..8c1a6be6 100644 --- a/pkg/assets/extract.go +++ b/pkg/assets/extract.go @@ -1,35 +1,37 @@ package assets import ( - "embed" "fmt" - "io/fs" "os" "path/filepath" + rice "github.com/GeertJohan/go.rice" "github.com/mudler/LocalAI/pkg/library" ) +const backendAssetsDir = "backend-assets" + func ResolvePath(dir string, paths ...string) string { - return filepath.Join(append([]string{dir, "backend-assets"}, paths...)...) + return filepath.Join(append([]string{dir, backendAssetsDir}, paths...)...) } -func ExtractFiles(content embed.FS, extractDir string) error { - // Create the target directory if it doesn't exist - err := os.MkdirAll(extractDir, 0750) +func ExtractFiles(content *rice.Box, extractDir string) error { + // Create the target directory with backend-assets subdirectory + backendAssetsDir := filepath.Join(extractDir, backendAssetsDir) + err := os.MkdirAll(backendAssetsDir, 0750) if err != nil { return fmt.Errorf("failed to create directory: %v", err) } - // Walk through the embedded FS and extract files - err = fs.WalkDir(content, ".", func(path string, d fs.DirEntry, err error) error { + // Walk through the rice box and extract files + err = content.Walk("", func(path string, info os.FileInfo, err error) error { if err != nil { return err } // Reconstruct the directory structure in the target directory - targetFile := filepath.Join(extractDir, path) - if d.IsDir() { + targetFile := filepath.Join(backendAssetsDir, path) + if info.IsDir() { // Create the directory in the target directory err := os.MkdirAll(targetFile, 0750) if err != nil { @@ -38,8 +40,8 @@ func ExtractFiles(content embed.FS, extractDir string) error { return nil } - // Read the file from the embedded FS - fileData, err := content.ReadFile(path) + // Read the file from the rice box + fileData, err := content.Bytes(path) if err != nil { return fmt.Errorf("failed to read file: %v", err) } @@ -56,7 +58,7 @@ func ExtractFiles(content embed.FS, extractDir string) error { // If there is a lib directory, set LD_LIBRARY_PATH to include it // we might use this mechanism to carry over e.g. Nvidia CUDA libraries // from the embedded FS to the target directory - library.LoadExtractedLibs(extractDir) + library.LoadExtractedLibs(backendAssetsDir) return err } diff --git a/pkg/assets/list.go b/pkg/assets/list.go index 47e60a40..edfdf498 100644 --- a/pkg/assets/list.go +++ b/pkg/assets/list.go @@ -1,19 +1,19 @@ package assets import ( - "embed" - "io/fs" + "os" + rice "github.com/GeertJohan/go.rice" "github.com/rs/zerolog/log" ) -func ListFiles(content embed.FS) (files []string) { - err := fs.WalkDir(content, ".", func(path string, d fs.DirEntry, err error) error { +func ListFiles(content *rice.Box) (files []string) { + err := content.Walk("", func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if d.IsDir() { + if info.IsDir() { return nil } @@ -21,7 +21,7 @@ func ListFiles(content embed.FS) (files []string) { return nil }) if err != nil { - log.Error().Err(err).Msg("error walking the embedded filesystem") + log.Error().Err(err).Msg("error walking the rice box") } return } diff --git a/pkg/library/dynaload.go b/pkg/library/dynaload.go index c1f79f65..878cdc88 100644 --- a/pkg/library/dynaload.go +++ b/pkg/library/dynaload.go @@ -25,7 +25,7 @@ func LoadExtractedLibs(dir string) error { } var err error = nil - for _, libDir := range []string{filepath.Join(dir, "backend-assets", "lib"), filepath.Join(dir, "lib")} { + for _, libDir := range []string{filepath.Join(dir, "lib"), filepath.Join(dir, "lib")} { err = errors.Join(err, LoadExternal(libDir)) } return err