dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-21 11:04:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

feat: allow to specify a reply prefix (#4931)

Browse source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto 2025-03-02 16:07:32 +01:00 • committed by GitHub
parent ff85f01459
commit a7b4001b75
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 38 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

12
core/backend/llm.go
View file

@ -116,6 +116,11 @@ func ModelInference(ctx context.Context, s string, messages []schema.Message, im
} }
if tokenCallback != nil { if tokenCallback != nil {
if c.TemplateConfig.ReplyPrefix != "" {
tokenCallback(c.TemplateConfig.ReplyPrefix, tokenUsage)
}
ss := "" ss := ""
var partialRune []byte var partialRune []byte
@ -165,8 +170,13 @@ func ModelInference(ctx context.Context, s string, messages []schema.Message, im
tokenUsage.TimingTokenGeneration = reply.TimingTokenGeneration tokenUsage.TimingTokenGeneration = reply.TimingTokenGeneration
tokenUsage.TimingPromptProcessing = reply.TimingPromptProcessing tokenUsage.TimingPromptProcessing = reply.TimingPromptProcessing
response := string(reply.Message)
if c.TemplateConfig.ReplyPrefix != "" {
response = c.TemplateConfig.ReplyPrefix + response
}
return LLMResponse{ return LLMResponse{
Response: string(reply.Message), Response: response,
Usage: tokenUsage, Usage: tokenUsage,
}, err }, err
} }

2
core/config/backend_config.go
View file

@ -213,6 +213,8 @@ type TemplateConfig struct {
Multimodal string `yaml:"multimodal"` Multimodal string `yaml:"multimodal"`
JinjaTemplate bool `yaml:"jinja_template"` JinjaTemplate bool `yaml:"jinja_template"`
ReplyPrefix string `yaml:"reply_prefix"`
} }
func (c *BackendConfig) UnmarshalYAML(value *yaml.Node) error { func (c *BackendConfig) UnmarshalYAML(value *yaml.Node) error {