dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-30 06:30:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add machine tag option, add extraUsage option, grpc-server -> proto -> endpoint extraUsage data is broken for now

Browse source 
Signed-off-by: mintyleaf <mintyleafdev@gmail.com>
This commit is contained in:
mintyleaf 2025-01-09 04:49:57 +04:00
parent 20edd44463
commit f040aa46a3
19 changed files with 192 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

18
backend/cpp/llama/grpc-server.cpp
View file

@ -2414,6 +2414,15 @@ public:
int32_t tokens_evaluated = result.result_json.value("tokens_evaluated", 0);
reply.set_prompt_tokens(tokens_evaluated);
int32_t timing_prompt_tokens = result.result_json.value("timings", json{}).value("prompt_n", 0);
reply.set_timing_prompt_tokens(timing_prompt_tokens);
int32_t timing_predicted_tokens = result.result_json.value("timings", json{}).value("predicted_n", 0);
reply.set_timing_predicted_tokens(timing_predicted_tokens);
double timing_prompt_processing = result.result_json.value("timings", json{}).value("prompt_ms", 0.0);
reply.set_timing_prompt_processing(timing_prompt_processing);
double timing_token_generation = result.result_json.value("timings", json{}).value("predicted_ms", 0.0);
reply.set_timing_token_generation(timing_token_generation);
// Log Request Correlation Id
LOG_VERBOSE("correlation:", {
{ "id", data["correlation_id"] }
@ -2454,6 +2463,15 @@ public:
reply->set_prompt_tokens(tokens_evaluated);
reply->set_tokens(tokens_predicted);
reply->set_message(completion_text);
int32_t timing_prompt_tokens = result.result_json.value("timings", json{}).value("prompt_n", 0);
reply->set_timing_prompt_tokens(timing_prompt_tokens);
int32_t timing_predicted_tokens = result.result_json.value("timings", json{}).value("predicted_n", 0);
reply->set_timing_predicted_tokens(timing_predicted_tokens);
double timing_prompt_processing = result.result_json.value("timings", json{}).value("prompt_ms", 0.0);
reply->set_timing_prompt_processing(timing_prompt_processing);
double timing_token_generation = result.result_json.value("timings", json{}).value("predicted_ms", 0.0);
reply->set_timing_token_generation(timing_token_generation);
}
else
{