dearwolf/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-30 06:30:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

remove redurant timing fields, fix not working timings output

Browse source 
Signed-off-by: mintyleaf <mintyleafdev@gmail.com>
This commit is contained in:
mintyleaf 2025-01-10 04:25:13 +04:00
parent f040aa46a3
commit 625029cc96
8 changed files with 16 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

28
backend/cpp/llama/grpc-server.cpp
View file

@ -2414,14 +2414,12 @@ public:
int32_t tokens_evaluated = result.result_json.value("tokens_evaluated", 0);
reply.set_prompt_tokens(tokens_evaluated);
int32_t timing_prompt_tokens = result.result_json.value("timings", json{}).value("prompt_n", 0);
reply.set_timing_prompt_tokens(timing_prompt_tokens);
int32_t timing_predicted_tokens = result.result_json.value("timings", json{}).value("predicted_n", 0);
reply.set_timing_predicted_tokens(timing_predicted_tokens);
double timing_prompt_processing = result.result_json.value("timings", json{}).value("prompt_ms", 0.0);
reply.set_timing_prompt_processing(timing_prompt_processing);
double timing_token_generation = result.result_json.value("timings", json{}).value("predicted_ms", 0.0);
reply.set_timing_token_generation(timing_token_generation);
if (result.result_json.contains("timings")) {
double timing_prompt_processing = result.result_json.at("timings").value("prompt_ms", 0.0);
reply.set_timing_prompt_processing(timing_prompt_processing);
double timing_token_generation = result.result_json.at("timings").value("predicted_ms", 0.0);
reply.set_timing_token_generation(timing_token_generation);
}
// Log Request Correlation Id
LOG_VERBOSE("correlation:", {
@ -2464,14 +2462,12 @@ public:
reply->set_tokens(tokens_predicted);
reply->set_message(completion_text);
int32_t timing_prompt_tokens = result.result_json.value("timings", json{}).value("prompt_n", 0);
reply->set_timing_prompt_tokens(timing_prompt_tokens);
int32_t timing_predicted_tokens = result.result_json.value("timings", json{}).value("predicted_n", 0);
reply->set_timing_predicted_tokens(timing_predicted_tokens);
double timing_prompt_processing = result.result_json.value("timings", json{}).value("prompt_ms", 0.0);
reply->set_timing_prompt_processing(timing_prompt_processing);
double timing_token_generation = result.result_json.value("timings", json{}).value("predicted_ms", 0.0);
reply->set_timing_token_generation(timing_token_generation);
if (result.result_json.contains("timings")) {
double timing_prompt_processing = result.result_json.at("timings").value("prompt_ms", 0.0);
reply->set_timing_prompt_processing(timing_prompt_processing);
double timing_token_generation = result.result_json.at("timings").value("predicted_ms", 0.0);
reply->set_timing_token_generation(timing_token_generation);
}
}
else
{