feat: kong cli refactor fixes #1955 (#1974)

* feat: migrate to alecthomas/kong for CLI Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * feat: bring in new flag for granular log levels Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * chore: go mod tidy Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * feat: allow loading cli flag values from ["./localai.yaml", "~/.config/localai.yaml", "/etc/localai.yaml"] in that order Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * feat: load from .env file instead of a yaml file Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * feat: better loading for environment files Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * feat(doc): add initial documentation about configuration Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: remove test log lines Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * feat: integrate new documentation into existing pages Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * feat: add documentation on .env files Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: cleanup some documentation table errors Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * feat: refactor CLI logic out to it's own package under core/cli Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> --------- Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com>
2025-05-27 22:15:00 +00:00 · 2024-04-11 02:19:24 -05:00 · 2024-04-11 02:19:24 -05:00 · 24d7dadfed
commit 24d7dadfed
parent 92005b9c02
10 changed files with 552 additions and 623 deletions
--- a/core/cli/cli.go
+++ b/core/cli/cli.go
@ -0,0 +1,20 @@
+package cli
+
+import "embed"
+
+type Context struct {
+	Debug    bool    `env:"LOCALAI_DEBUG,DEBUG" default:"false" hidden:"" help:"DEPRECATED, use --log-level=debug instead. Enable debug logging"`
+	LogLevel *string `env:"LOCALAI_LOG_LEVEL" enum:"error,warn,info,debug" help:"Set the level of logs to output [${enum}]"`
+
+	// This field is not a command line argument/flag, the struct tag excludes it from the parsed CLI
+	BackendAssets embed.FS `kong:"-"`
+}
+
+var CLI struct {
+	Context `embed:""`
+
+	Run        RunCMD        `cmd:"" help:"Run LocalAI, this the default command if no other command is specified. Run 'local-ai run --help' for more information" default:"withargs"`
+	Models     ModelsCMD     `cmd:"" help:"Manage LocalAI models and definitions"`
+	TTS        TTSCMD        `cmd:"" help:"Convert text to speech"`
+	Transcript TranscriptCMD `cmd:"" help:"Convert audio to text"`
+}
--- a/core/cli/models.go
+++ b/core/cli/models.go
@ -0,0 +1,74 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/go-skynet/LocalAI/pkg/gallery"
+	"github.com/rs/zerolog/log"
+	"github.com/schollz/progressbar/v3"
+)
+
+type ModelsCMDFlags struct {
+	Galleries  string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models"`
+	ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
+}
+
+type ModelsList struct {
+	ModelsCMDFlags `embed:""`
+}
+
+type ModelsInstall struct {
+	ModelArgs []string `arg:"" optional:"" name:"models" help:"Model configuration URLs to load"`
+
+	ModelsCMDFlags `embed:""`
+}
+
+type ModelsCMD struct {
+	List    ModelsList    `cmd:"" help:"List the models avaiable in your galleries" default:"withargs"`
+	Install ModelsInstall `cmd:"" help:"Install a model from the gallery"`
+}
+
+func (ml *ModelsList) Run(ctx *Context) error {
+	var galleries []gallery.Gallery
+	if err := json.Unmarshal([]byte(ml.Galleries), &galleries); err != nil {
+		log.Error().Err(err).Msg("unable to load galleries")
+	}
+
+	models, err := gallery.AvailableGalleryModels(galleries, ml.ModelsPath)
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		if model.Installed {
+			fmt.Printf(" * %s@%s (installed)\n", model.Gallery.Name, model.Name)
+		} else {
+			fmt.Printf(" - %s@%s\n", model.Gallery.Name, model.Name)
+		}
+	}
+	return nil
+}
+
+func (mi *ModelsInstall) Run(ctx *Context) error {
+	modelName := mi.ModelArgs[0]
+
+	var galleries []gallery.Gallery
+	if err := json.Unmarshal([]byte(mi.Galleries), &galleries); err != nil {
+		log.Error().Err(err).Msg("unable to load galleries")
+	}
+
+	progressBar := progressbar.NewOptions(
+		1000,
+		progressbar.OptionSetDescription(fmt.Sprintf("downloading model %s", modelName)),
+		progressbar.OptionShowBytes(false),
+		progressbar.OptionClearOnFinish(),
+	)
+	progressCallback := func(fileName string, current string, total string, percentage float64) {
+		progressBar.Set(int(percentage * 10))
+	}
+	err := gallery.InstallModelFromGallery(galleries, modelName, mi.ModelsPath, gallery.GalleryModel{}, progressCallback)
+	if err != nil {
+		return err
+	}
+	return nil
+}
--- a/core/cli/run.go
+++ b/core/cli/run.go
@ -0,0 +1,155 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/go-skynet/LocalAI/core/config"
+	"github.com/go-skynet/LocalAI/core/http"
+	"github.com/go-skynet/LocalAI/core/startup"
+	"github.com/rs/zerolog/log"
+)
+
+type RunCMD struct {
+	ModelArgs []string `arg:"" optional:"" name:"models" help:"Model configuration URLs to load"`
+
+	ModelsPath        string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
+	BackendAssetsPath string `env:"LOCALAI_BACKEND_ASSETS_PATH,BACKEND_ASSETS_PATH" type:"path" default:"/tmp/localai/backend_data" help:"Path used to extract libraries that are required by some of the backends in runtime" group:"storage"`
+	ImagePath         string `env:"LOCALAI_IMAGE_PATH,IMAGE_PATH" type:"path" default:"/tmp/generated/images" help:"Location for images generated by backends (e.g. stablediffusion)" group:"storage"`
+	AudioPath         string `env:"LOCALAI_AUDIO_PATH,AUDIO_PATH" type:"path" default:"/tmp/generated/audio" help:"Location for audio generated by backends (e.g. piper)" group:"storage"`
+	UploadPath        string `env:"LOCALAI_UPLOAD_PATH,UPLOAD_PATH" type:"path" default:"/tmp/localai/upload" help:"Path to store uploads from files api" group:"storage"`
+	ConfigPath        string `env:"LOCALAI_CONFIG_PATH,CONFIG_PATH" default:"/tmp/localai/config" group:"storage"`
+	LocalaiConfigDir  string `env:"LOCALAI_CONFIG_DIR" type:"path" default:"${basepath}/configuration" help:"Directory for dynamic loading of certain configuration files (currently api_keys.json and external_backends.json)" group:"storage"`
+	// The alias on this option is there to preserve functionality with the old `--config-file` parameter
+	ModelsConfigFile string `env:"LOCALAI_MODELS_CONFIG_FILE,CONFIG_FILE" aliases:"config-file" help:"YAML file containing a list of model backend configs" group:"storage"`
+
+	Galleries           string   `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models"`
+	AutoloadGalleries   bool     `env:"LOCALAI_AUTOLOAD_GALLERIES,AUTOLOAD_GALLERIES" group:"models"`
+	RemoteLibrary       string   `env:"LOCALAI_REMOTE_LIBRARY,REMOTE_LIBRARY" default:"${remoteLibraryURL}" help:"A LocalAI remote library URL" group:"models"`
+	PreloadModels       string   `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"`
+	Models              []string `env:"LOCALAI_MODELS,MODELS" help:"A List of model configuration URLs to load" group:"models"`
+	PreloadModelsConfig string   `env:"LOCALAI_PRELOAD_MODELS_CONFIG,PRELOAD_MODELS_CONFIG" help:"A List of models to apply at startup. Path to a YAML config file" group:"models"`
+
+	F16         bool `name:"f16" env:"LOCALAI_F16,F16" help:"Enable GPU acceleration" group:"performance"`
+	Threads     int  `env:"LOCALAI_THREADS,THREADS" short:"t" default:"4" help:"Number of threads used for parallel computation. Usage of the number of physical cores in the system is suggested" group:"performance"`
+	ContextSize int  `env:"LOCALAI_CONTEXT_SIZE,CONTEXT_SIZE" default:"512" help:"Default context size for models" group:"performance"`
+
+	Address          string   `env:"LOCALAI_ADDRESS,ADDRESS" default:":8080" help:"Bind address for the API server" group:"api"`
+	CORS             bool     `env:"LOCALAI_CORS,CORS" help:"" group:"api"`
+	CORSAllowOrigins string   `env:"LOCALAI_CORS_ALLOW_ORIGINS,CORS_ALLOW_ORIGINS" group:"api"`
+	UploadLimit      int      `env:"LOCALAI_UPLOAD_LIMIT,UPLOAD_LIMIT" default:"15" help:"Default upload-limit in MB" group:"api"`
+	APIKeys          []string `env:"LOCALAI_API_KEY,API_KEY" help:"List of API Keys to enable API authentication. When this is set, all the requests must be authenticated with one of these API keys" group:"api"`
+	DisableWelcome   bool     `env:"LOCALAI_DISABLE_WELCOME,DISABLE_WELCOME" default:"false" help:"Disable welcome pages" group:"api"`
+
+	ParallelRequests     bool     `env:"LOCALAI_PARALLEL_REQUESTS,PARALLEL_REQUESTS" help:"Enable backends to handle multiple requests in parallel if they support it (e.g.: llama.cpp or vllm)" group:"backends"`
+	SingleActiveBackend  bool     `env:"LOCALAI_SINGLE_ACTIVE_BACKEND,SINGLE_ACTIVE_BACKEND" help:"Allow only one backend to be run at a time" group:"backends"`
+	PreloadBackendOnly   bool     `env:"LOCALAI_PRELOAD_BACKEND_ONLY,PRELOAD_BACKEND_ONLY" default:"false" help:"Do not launch the API services, only the preloaded models / backends are started (useful for multi-node setups)" group:"backends"`
+	ExternalGRPCBackends []string `env:"LOCALAI_EXTERNAL_GRPC_BACKENDS,EXTERNAL_GRPC_BACKENDS" help:"A list of external grpc backends" group:"backends"`
+	EnableWatchdogIdle   bool     `env:"LOCALAI_WATCHDOG_IDLE,WATCHDOG_IDLE" default:"false" help:"Enable watchdog for stopping backends that are idle longer than the watchdog-idle-timeout" group:"backends"`
+	WatchdogIdleTimeout  string   `env:"LOCALAI_WATCHDOG_IDLE_TIMEOUT,WATCHDOG_IDLE_TIMEOUT" default:"15m" help:"Threshold beyond which an idle backend should be stopped" group:"backends"`
+	EnableWatchdogBusy   bool     `env:"LOCALAI_WATCHDOG_BUSY,WATCHDOG_BUSY" default:"false" help:"Enable watchdog for stopping backends that are busy longer than the watchdog-busy-timeout" group:"backends"`
+	WatchdogBusyTimeout  string   `env:"LOCALAI_WATCHDOG_BUSY_TIMEOUT,WATCHDOG_BUSY_TIMEOUT" default:"5m" help:"Threshold beyond which a busy backend should be stopped" group:"backends"`
+}
+
+func (r *RunCMD) Run(ctx *Context) error {
+	opts := []config.AppOption{
+		config.WithConfigFile(r.ModelsConfigFile),
+		config.WithJSONStringPreload(r.PreloadModels),
+		config.WithYAMLConfigPreload(r.PreloadModelsConfig),
+		config.WithModelPath(r.ModelsPath),
+		config.WithContextSize(r.ContextSize),
+		config.WithDebug(ctx.Debug),
+		config.WithImageDir(r.ImagePath),
+		config.WithAudioDir(r.AudioPath),
+		config.WithUploadDir(r.UploadPath),
+		config.WithConfigsDir(r.ConfigPath),
+		config.WithF16(r.F16),
+		config.WithStringGalleries(r.Galleries),
+		config.WithModelLibraryURL(r.RemoteLibrary),
+		config.WithDisableMessage(false),
+		config.WithCors(r.CORS),
+		config.WithCorsAllowOrigins(r.CORSAllowOrigins),
+		config.WithThreads(r.Threads),
+		config.WithBackendAssets(ctx.BackendAssets),
+		config.WithBackendAssetsOutput(r.BackendAssetsPath),
+		config.WithUploadLimitMB(r.UploadLimit),
+		config.WithApiKeys(r.APIKeys),
+		config.WithModelsURL(append(r.Models, r.ModelArgs...)...),
+	}
+
+	idleWatchDog := r.EnableWatchdogIdle
+	busyWatchDog := r.EnableWatchdogBusy
+
+	if r.DisableWelcome {
+		opts = append(opts, config.DisableWelcomePage)
+	}
+
+	if idleWatchDog || busyWatchDog {
+		opts = append(opts, config.EnableWatchDog)
+		if idleWatchDog {
+			opts = append(opts, config.EnableWatchDogIdleCheck)
+			dur, err := time.ParseDuration(r.WatchdogIdleTimeout)
+			if err != nil {
+				return err
+			}
+			opts = append(opts, config.SetWatchDogIdleTimeout(dur))
+		}
+		if busyWatchDog {
+			opts = append(opts, config.EnableWatchDogBusyCheck)
+			dur, err := time.ParseDuration(r.WatchdogBusyTimeout)
+			if err != nil {
+				return err
+			}
+			opts = append(opts, config.SetWatchDogBusyTimeout(dur))
+		}
+	}
+	if r.ParallelRequests {
+		opts = append(opts, config.EnableParallelBackendRequests)
+	}
+	if r.SingleActiveBackend {
+		opts = append(opts, config.EnableSingleBackend)
+	}
+
+	// split ":" to get backend name and the uri
+	for _, v := range r.ExternalGRPCBackends {
+		backend := v[:strings.IndexByte(v, ':')]
+		uri := v[strings.IndexByte(v, ':')+1:]
+		opts = append(opts, config.WithExternalBackend(backend, uri))
+	}
+
+	if r.AutoloadGalleries {
+		opts = append(opts, config.EnableGalleriesAutoload)
+	}
+
+	if r.PreloadBackendOnly {
+		_, _, _, err := startup.Startup(opts...)
+		return err
+	}
+
+	cl, ml, options, err := startup.Startup(opts...)
+
+	if err != nil {
+		return fmt.Errorf("failed basic startup tasks with error %s", err.Error())
+	}
+
+	// Watch the configuration directory
+	// If the directory does not exist, we don't watch it
+	if _, err := os.Stat(r.LocalaiConfigDir); err == nil {
+		closeConfigWatcherFn, err := startup.WatchConfigDirectory(r.LocalaiConfigDir, options)
+		defer closeConfigWatcherFn()
+
+		if err != nil {
+			return fmt.Errorf("failed while watching configuration directory %s", r.LocalaiConfigDir)
+		}
+	}
+
+	appHTTP, err := http.App(cl, ml, options)
+	if err != nil {
+		log.Error().Err(err).Msg("error during HTTP App construction")
+		return err
+	}
+
+	return appHTTP.Listen(r.Address)
+}
--- a/core/cli/transcript.go
+++ b/core/cli/transcript.go
@ -0,0 +1,54 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/go-skynet/LocalAI/core/backend"
+	"github.com/go-skynet/LocalAI/core/config"
+	"github.com/go-skynet/LocalAI/pkg/model"
+)
+
+type TranscriptCMD struct {
+	Filename string `arg:""`
+
+	Backend           string `short:"b" default:"whisper" help:"Backend to run the transcription model"`
+	Model             string `short:"m" required:"" help:"Model name to run the TTS"`
+	Language          string `short:"l" help:"Language of the audio file"`
+	Threads           int    `short:"t" default:"1" help:"Number of threads used for parallel computation"`
+	ModelsPath        string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
+	BackendAssetsPath string `env:"LOCALAI_BACKEND_ASSETS_PATH,BACKEND_ASSETS_PATH" type:"path" default:"/tmp/localai/backend_data" help:"Path used to extract libraries that are required by some of the backends in runtime" group:"storage"`
+}
+
+func (t *TranscriptCMD) Run(ctx *Context) error {
+	opts := &config.ApplicationConfig{
+		ModelPath:         t.ModelsPath,
+		Context:           context.Background(),
+		AssetsDestination: t.BackendAssetsPath,
+	}
+
+	cl := config.NewBackendConfigLoader()
+	ml := model.NewModelLoader(opts.ModelPath)
+	if err := cl.LoadBackendConfigsFromPath(t.ModelsPath); err != nil {
+		return err
+	}
+
+	c, exists := cl.GetBackendConfig(t.Model)
+	if !exists {
+		return errors.New("model not found")
+	}
+
+	c.Threads = &t.Threads
+
+	defer ml.StopAllGRPC()
+
+	tr, err := backend.ModelTranscription(t.Filename, t.Language, ml, c, opts)
+	if err != nil {
+		return err
+	}
+	for _, segment := range tr.Segments {
+		fmt.Println(segment.Start.String(), "-", segment.Text)
+	}
+	return nil
+}
--- a/core/cli/tts.go
+++ b/core/cli/tts.go
@ -0,0 +1,61 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/go-skynet/LocalAI/core/backend"
+	"github.com/go-skynet/LocalAI/core/config"
+	"github.com/go-skynet/LocalAI/pkg/model"
+)
+
+type TTSCMD struct {
+	Text []string `arg:""`
+
+	Backend           string `short:"b" default:"piper" help:"Backend to run the TTS model"`
+	Model             string `short:"m" required:"" help:"Model name to run the TTS"`
+	Voice             string `short:"v" help:"Voice name to run the TTS"`
+	OutputFile        string `short:"o" type:"path" help:"The path to write the output wav file"`
+	ModelsPath        string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
+	BackendAssetsPath string `env:"LOCALAI_BACKEND_ASSETS_PATH,BACKEND_ASSETS_PATH" type:"path" default:"/tmp/localai/backend_data" help:"Path used to extract libraries that are required by some of the backends in runtime" group:"storage"`
+}
+
+func (t *TTSCMD) Run(ctx *Context) error {
+	outputFile := t.OutputFile
+	outputDir := t.BackendAssetsPath
+	if outputFile != "" {
+		outputDir = filepath.Dir(outputFile)
+	}
+
+	text := strings.Join(t.Text, " ")
+
+	opts := &config.ApplicationConfig{
+		ModelPath:         t.ModelsPath,
+		Context:           context.Background(),
+		AudioDir:          outputDir,
+		AssetsDestination: t.BackendAssetsPath,
+	}
+	ml := model.NewModelLoader(opts.ModelPath)
+
+	defer ml.StopAllGRPC()
+
+	options := config.BackendConfig{}
+	options.SetDefaults()
+
+	filePath, _, err := backend.ModelTTS(t.Backend, text, t.Model, t.Voice, ml, opts, options)
+	if err != nil {
+		return err
+	}
+	if outputFile != "" {
+		if err := os.Rename(filePath, outputFile); err != nil {
+			return err
+		}
+		fmt.Printf("Generate file %s\n", outputFile)
+	} else {
+		fmt.Printf("Generate file %s\n", filePath)
+	}
+	return nil
+}