dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-31 07:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 34

Revert #1963 (#2056)

Browse source 
* Revert "fix(fncall): fix regression introduced in #1963 (#2048)"

This reverts commit 6b06d4e0af.

* Revert "fix: action-tmate back to upstream, dead code removal (#2038)"

This reverts commit fdec8a9d00.

* Revert "feat(grpc): return consumed token count and update response accordingly (#2035)"

This reverts commit e843d7df0e.

* Revert "refactor: backend/service split, channel-based llm flow (#1963)"

This reverts commit eed5706994.

* feat(grpc): return consumed token count and update response accordingly

Fixes: #1920

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto 2024-04-17 23:33:49 +02:00 • committed by GitHub
parent af8c705ecd
commit af9e5a2d05
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
52 changed files with 2295 additions and 3065 deletions
Download patch file Download diff file Expand all files Collapse all files

65
core/http/ctx/fiber.go
View file

@ -1,88 +1,43 @@
package fiberContext
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/go-skynet/LocalAI/core/config"
"github.com/go-skynet/LocalAI/core/schema"
"github.com/go-skynet/LocalAI/pkg/model"
"github.com/gofiber/fiber/v2"
"github.com/rs/zerolog/log"
)
type FiberContextExtractor struct {
ml *model.ModelLoader
appConfig *config.ApplicationConfig
}
func NewFiberContextExtractor(ml *model.ModelLoader, appConfig *config.ApplicationConfig) *FiberContextExtractor {
return &FiberContextExtractor{
ml: ml,
appConfig: appConfig,
}
}
// ModelFromContext returns the model from the context
// If no model is specified, it will take the first available
// Takes a model string as input which should be the one received from the user request.
// It returns the model name resolved from the context and an error if any.
func (fce *FiberContextExtractor) ModelFromContext(ctx *fiber.Ctx, modelInput string, firstModel bool) (string, error) {
ctxPM := ctx.Params("model")
if ctxPM != "" {
log.Debug().Msgf("[FCE] Overriding param modelInput %q with ctx.Params value %q", modelInput, ctxPM)
modelInput = ctxPM
func ModelFromContext(ctx *fiber.Ctx, loader *model.ModelLoader, modelInput string, firstModel bool) (string, error) {
if ctx.Params("model") != "" {
modelInput = ctx.Params("model")
}
// Set model from bearer token, if available
bearer := strings.TrimPrefix(ctx.Get("authorization"), "Bearer ")
bearerExists := bearer != "" && fce.ml.ExistsInModelPath(bearer)
bearer := strings.TrimLeft(ctx.Get("authorization"), "Bearer ")
bearerExists := bearer != "" && loader.ExistsInModelPath(bearer)
// If no model was specified, take the first available
if modelInput == "" && !bearerExists && firstModel {
models, _ := fce.ml.ListModels()
models, _ := loader.ListModels()
if len(models) > 0 {
modelInput = models[0]
log.Debug().Msgf("[FCE] No model specified, using first available: %s", modelInput)
log.Debug().Msgf("No model specified, using: %s", modelInput)
} else {
log.Warn().Msgf("[FCE] No model specified, none available")
return "", fmt.Errorf("[fce] no model specified, none available")
log.Debug().Msgf("No model specified, returning error")
return "", fmt.Errorf("no model specified")
}
}
// If a model is found in bearer token takes precedence
if bearerExists {
log.Debug().Msgf("[FCE] Using model from bearer token: %s", bearer)
log.Debug().Msgf("Using model from bearer token: %s", bearer)
modelInput = bearer
}
if modelInput == "" {
log.Warn().Msg("[FCE] modelInput is empty")
}
return modelInput, nil
}
// TODO: Do we still need the first return value?
func (fce *FiberContextExtractor) OpenAIRequestFromContext(c *fiber.Ctx, firstModel bool) (string, *schema.OpenAIRequest, error) {
input := new(schema.OpenAIRequest)
// Get input data from the request body
if err := c.BodyParser(input); err != nil {
return "", nil, fmt.Errorf("failed parsing request body: %w", err)
}
received, _ := json.Marshal(input)
ctx, cancel := context.WithCancel(fce.appConfig.Context)
input.Context = ctx
input.Cancel = cancel
log.Debug().Msgf("Request received: %s", string(received))
var err error
input.Model, err = fce.ModelFromContext(c, input.Model, firstModel)
return input.Model, input, err
}