diff --git a/benchmark/swe-bench-lite.txt b/benchmark/swe-bench-lite.txt
new file mode 100644
index 000000000..b61c44c30
--- /dev/null
+++ b/benchmark/swe-bench-lite.txt
@@ -0,0 +1,44 @@
+model: gpt-3.5-turbo
+validate_environment: {'keys_in_environment': True, 'missing_keys': []}
+completion: Hello! How can I assist you today?
+
+model: claude-3-sonnet-20240229
+validate_environment: {'keys_in_environment': True, 'missing_keys': []}
+completion: Hello! How can I assist you today?
+
+model: gemini/gemini-1.5-pro-latest
+validate_environment: {'keys_in_environment': False, 'missing_keys': []}
+completion: Hello! 👋 How can I help you today? 😊 
+
+
+model: command-r-plus
+validate_environment: {'keys_in_environment': False, 'missing_keys': []}
+completion: Hello! How can I help you today?
+
+erased environment
+
+model: gpt-3.5-turbo
+validate_environment: {'keys_in_environment': False, 'missing_keys': ['OPENAI_API_KEY']}
+completion error: OpenAIException - Traceback (most recent call last):
+  File "/Users/gauthier/Projects/aider/.venv/lib/python3.11/site-packages/litellm/llms/openai.py", line 414, in completion
+    raise e
+  File "/Users/gauthier/Projects/aider/.venv/lib/python3.11/site-packages/litellm/llms/openai.py", line 350, in completion
+    openai_client = openai(
+                    ^^^^^^^
+  File "/Users/gauthier/Projects/aider/.venv/lib/python3.11/site-packages/openai/_client.py", line 104, in __init__
+    raise openaiError(
+openai.openaiError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable
+
+
+model: claude-3-sonnet-20240229
+validate_environment: {'keys_in_environment': False, 'missing_keys': ['ANTHROPIC_API_KEY']}
+completion error: Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params
+
+model: gemini/gemini-1.5-pro-latest
+validate_environment: {'keys_in_environment': False, 'missing_keys': []}
+completion error: PalmException - Invalid api key
+
+model: command-r-plus
+validate_environment: {'keys_in_environment': False, 'missing_keys': []}
+completion error: {"message":"no api key supplied"}
+
diff --git a/benchmark/swe_bench_lite.py b/benchmark/swe_bench_lite.py
index 72106f197..afa76c1d4 100644
--- a/benchmark/swe_bench_lite.py
+++ b/benchmark/swe_bench_lite.py
@@ -68,4 +68,4 @@ def plot_swe_bench_lite(data_file):
 
 
 # Example usage
-plot_swe_bench_lite("benchmark/tmp.txt")
+plot_swe_bench_lite("benchmark/swe-bench-lite.txt")