dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-05-20 02:24:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 21

feat(llama.cpp): expose cache_type_k and cache_type_v for quant of kv cache (#4329)

Browse source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto 2024-12-06 10:23:59 +01:00 • committed by GitHub
parent 88737e1d76
commit d4c1746c7d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 15 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
backend/cpp/llama/grpc-server.cpp
View file

@ -2241,6 +2241,12 @@ static void params_parse(const backend::ModelOptions* request,
}
// params.model_alias ??
params.model_alias = request->modelfile();
if (!request->cachetypekey().empty()) {
params.cache_type_k = request->cachetypekey();
}
if (!request->cachetypevalue().empty()) {
params.cache_type_v = request->cachetypevalue();
}
params.n_ctx = request->contextsize();
//params.memory_f16 = request->f16memory();
params.cpuparams.n_threads = request->threads();