From bc8f648a91a2f07b061b3ae1ee1944131c61cda2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Mon, 11 Mar 2024 19:49:03 +0100 Subject: [PATCH] fix(doc/examples): set defaults to mirostat (#1820) The default sampler on some models don't return enough candidates which leads to a false sense of randomness. Tracing back the code it looks that with the temperature sampler there might not be enough candidates to pick from, and since the seed and "randomness" take effect while picking a good candidate this yields to the same results over and over. Fixes https://github.com/mudler/LocalAI/issues/1723 by updating the examples and documentation to use mirostat instead. --- docs/content/docs/advanced/advanced-usage.md | 7 ++++--- embedded/models/codellama-7b-gguf.yaml | 6 +++++- embedded/models/dolphin-2.5-mixtral-8x7b.yaml | 5 ++++- embedded/models/llava.yaml | 3 +++ embedded/models/mistral-openorca.yaml | 4 ++++ embedded/models/mixtral-instruct.yaml | 4 ++++ examples/configurations/phi-2.yaml | 4 ++++ 7 files changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/content/docs/advanced/advanced-usage.md b/docs/content/docs/advanced/advanced-usage.md index a892cc36..c9926bab 100644 --- a/docs/content/docs/advanced/advanced-usage.md +++ b/docs/content/docs/advanced/advanced-usage.md @@ -130,13 +130,14 @@ parameters: typical_p: tfz: frequency_penalty: - mirostat_eta: - mirostat_tau: - mirostat: + rope_freq_base: rope_freq_scale: negative_prompt_scale: +mirostat_eta: +mirostat_tau: +mirostat: # Default context size context_size: 512 # Default number of threads diff --git a/embedded/models/codellama-7b-gguf.yaml b/embedded/models/codellama-7b-gguf.yaml index d2a6b518..413c838b 100644 --- a/embedded/models/codellama-7b-gguf.yaml +++ b/embedded/models/codellama-7b-gguf.yaml @@ -2,10 +2,14 @@ name: codellama-7b-gguf backend: transformers parameters: model: huggingface://TheBloke/CodeLlama-7B-GGUF/codellama-7b.Q4_K_M.gguf - temperature: 0.2 + temperature: 0.5 top_k: 40 seed: -1 top_p: 0.95 +mirostat: 2 +mirostat_eta: 1.0 +mirostat_tau: 1.0 + context_size: 4096 f16: true gpu_layers: 90 diff --git a/embedded/models/dolphin-2.5-mixtral-8x7b.yaml b/embedded/models/dolphin-2.5-mixtral-8x7b.yaml index ba6020c0..12ee1efc 100644 --- a/embedded/models/dolphin-2.5-mixtral-8x7b.yaml +++ b/embedded/models/dolphin-2.5-mixtral-8x7b.yaml @@ -2,10 +2,13 @@ name: dolphin-mixtral-8x7b mmap: true parameters: model: huggingface://TheBloke/dolphin-2.5-mixtral-8x7b-GGUF/dolphin-2.5-mixtral-8x7b.Q2_K.gguf - temperature: 0.2 + temperature: 0.5 top_k: 40 top_p: 0.95 seed: -1 +mirostat: 2 +mirostat_eta: 1.0 +mirostat_tau: 1.0 template: chat_message: | <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}} diff --git a/embedded/models/llava.yaml b/embedded/models/llava.yaml index 2e571f21..3d240681 100644 --- a/embedded/models/llava.yaml +++ b/embedded/models/llava.yaml @@ -18,6 +18,9 @@ parameters: top_k: 40 top_p: 0.95 seed: -1 +mirostat: 2 +mirostat_eta: 1.0 +mirostat_tau: 1.0 template: chat: | diff --git a/embedded/models/mistral-openorca.yaml b/embedded/models/mistral-openorca.yaml index f40d854f..0794a69b 100644 --- a/embedded/models/mistral-openorca.yaml +++ b/embedded/models/mistral-openorca.yaml @@ -6,6 +6,10 @@ parameters: top_k: 40 top_p: 0.95 seed: -1 +mirostat: 2 +mirostat_eta: 1.0 +mirostat_tau: 1.0 + template: chat_message: | <|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}} diff --git a/embedded/models/mixtral-instruct.yaml b/embedded/models/mixtral-instruct.yaml index 3272557a..246b2324 100644 --- a/embedded/models/mixtral-instruct.yaml +++ b/embedded/models/mixtral-instruct.yaml @@ -6,6 +6,10 @@ parameters: top_k: 40 seed: -1 top_p: 0.95 +mirostat: 2 +mirostat_eta: 1.0 +mirostat_tau: 1.0 + template: chat: &chat | [INST] {{.Input}} [/INST] diff --git a/examples/configurations/phi-2.yaml b/examples/configurations/phi-2.yaml index cac1e9da..e5a13442 100644 --- a/examples/configurations/phi-2.yaml +++ b/examples/configurations/phi-2.yaml @@ -11,6 +11,10 @@ parameters: top_k: 40 top_p: 0.95 seed: -1 + +mirostat: 2 +mirostat_eta: 1.0 +mirostat_tau: 1.0 template: chat: &template |- Instruct: {{.Input}}