dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-24 12:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 12

feat: add ngqa and RMSNormEps parameters (#860)

Browse source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto 2023-08-03 00:51:08 +02:00 • committed by GitHub
parent 953e30814a
commit 5ca21ee398
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 138 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

8
pkg/grpc/llm/llama/llama.go
View file

@ -33,6 +33,14 @@ func (llm *LLM) Load(opts *pb.ModelOptions) error {
llama.WithRopeFreqScale(ropeFreqScale),
}
if opts.NGQA != 0 {
llamaOpts = append(llamaOpts, llama.WithGQA(int(opts.NGQA)))
}
if opts.RMSNormEps != 0 {
llamaOpts = append(llamaOpts, llama.WithRMSNormEPS(opts.RMSNormEps))
}
if opts.ContextSize != 0 {
llamaOpts = append(llamaOpts, llama.SetContext(int(opts.ContextSize)))
}