From 8be2f8086c9d39aab7adb5827cf618d4e60260b1 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 13:17:55 -0700 Subject: [PATCH 01/10] aider: Add a test case to reproduce the issue of trying to add a directory using the `/add` command. # Aider chat conversation: USER: add a test case to reproduce this issue: Trying to add just a directory makes the program throw the following and exit: > /add src/components Traceback (most recent call last): File "/home/palmerd/.local/bin/aider", line 8, in sys.exit(main()) ^^^^^^ File "/home/palmerd/.local/lib/python3.11/site-packages/aider/main.py", line 369, in main coder.run() File "/home/palmerd/.local/lib/python3.11/site-packages/aider/coders/base_coder.py", line 382, in run new_user_message = self.run_loop() ^^^^^^^^^^^^^^^ File "/home/palmerd/.local/lib/python3.11/site-packages/aider/coders/base_coder.py", line 446, in run_loop return self.commands.run(inp) ^^^^^^^^^^^^^^^^^^^^^^ File "/home/palmerd/.local/lib/python3.11/site-packages/aider/commands.py", line 60, in run return self.do_run(matching_commands[0][1:], rest_inp) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/palmerd/.local/lib/python3.11/site-packages/aider/commands.py", line 45, in do_run return cmd_method(args) ^^^^^^^^^^^^^^^^ File "/home/palmerd/.local/lib/python3.11/site-packages/aider/commands.py", line 264, in cmd_add content = self.io.read_text(abs_file_path) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/palmerd/.local/lib/python3.11/site-packages/aider/io.py", line 142, in read_text with open(str(filename), "r", encoding=self.encoding) as f: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ IsADirectoryError: [Errno 21] Is a directory: '/home/palmerd/vue-frontend/src/components' ASSISTANT: To reproduce this issue, we need to create a test case that tries to add a directory using the `/add` command. However, the code you've provided is not designed to handle test cases. Could you please provide the test suite or the file where the test cases are written so that I can add a new test case? USER: I added these *read-write* files: tests/test_commands.py ASSISTANT: Great, we can add a test case in the `tests/test_commands.py` file. This test case will attempt to add a directory and check if the appropriate error is handled. Here is the proposed change: ```python tests/test_commands.py <<<<<<< ORIGINAL def test_cmd_add_no_match(self): # Initialize the Commands and InputOutput objects io = InputOutput(pretty=False, yes=True) from aider.coders import Coder coder = Coder.create(models.GPT35, None, io, openai_api_key="deadbeef") commands = Commands(io, coder) # Call the cmd_add method with a non-existent file pattern commands.cmd_add("*.nonexistent") # Check if no files have been added to the chat session self.assertEqual(len(coder.abs_fnames), 0) ======= def test_cmd_add_no_match(self): # Initialize the Commands and InputOutput objects io = InputOutput(pretty=False, yes=True) from aider.coders import Coder coder = Coder.create(models.GPT35, None, io, openai_api_key="deadbeef") commands = Commands(io, coder) # Call the cmd_add method with a non-existent file pattern commands.cmd_add("*.nonexistent") # Check if no files have been added to the chat session self.assertEqual(len(coder.abs_fnames), 0) def test_cmd_add_directory(self): # Initialize the Commands and InputOutput objects io = InputOutput(pretty=False, yes=True) from aider.coders import Coder coder = Coder.create(models.GPT35, None, io, openai_api_key="deadbeef") commands = Commands(io, coder) # Create a directory os.mkdir("test_dir") # Call the cmd_add method with a directory with self.assertRaises(IsADirectoryError): commands.cmd_add("test_dir") >>>>>>> UPDATED ``` This new test case `test_cmd_add_directory` creates a directory named `test_dir` and attempts to add it using the `cmd_add` method. We expect an `IsADirectoryError` to be raised in this scenario. --- tests/test_commands.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index c43826419..1aa4fcba1 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -79,6 +79,21 @@ class TestCommands(TestCase): # Check if no files have been added to the chat session self.assertEqual(len(coder.abs_fnames), 0) + def test_cmd_add_directory(self): + # Initialize the Commands and InputOutput objects + io = InputOutput(pretty=False, yes=True) + from aider.coders import Coder + + coder = Coder.create(models.GPT35, None, io, openai_api_key="deadbeef") + commands = Commands(io, coder) + + # Create a directory + os.mkdir("test_dir") + + # Call the cmd_add method with a directory + with self.assertRaises(IsADirectoryError): + commands.cmd_add("test_dir") + def test_cmd_drop_with_glob_patterns(self): # Initialize the Commands and InputOutput objects io = InputOutput(pretty=False, yes=True) From 4b91adabdcf3c085d623a6b61f409fd6d6eb6bbe Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 13:19:11 -0700 Subject: [PATCH 02/10] aider: The test case has been modified to add files to the directory and ensure they are added correctly. # Aider chat conversation: USER: put some files in the test_dir, and the test shouldn't look for the exception. it should make sure the files were added ASSISTANT: To modify the test case to add files to the directory and ensure they are added correctly, we need to modify the `tests/test_commands.py` file. Here is the proposed change: ```python tests/test_commands.py <<<<<<< ORIGINAL def test_cmd_add_directory(self): # Initialize the Commands and InputOutput objects io = InputOutput(pretty=False, yes=True) from aider.coders import Coder coder = Coder.create(models.GPT35, None, io, openai_api_key="deadbeef") commands = Commands(io, coder) # Create a directory os.mkdir("test_dir") # Call the cmd_add method with a directory with self.assertRaises(IsADirectoryError): commands.cmd_add("test_dir") ======= def test_cmd_add_directory(self): # Initialize the Commands and InputOutput objects io = InputOutput(pretty=False, yes=True) from aider.coders import Coder coder = Coder.create(models.GPT35, None, io, openai_api_key="deadbeef") commands = Commands(io, coder) # Create a directory and add files to it os.mkdir("test_dir") with open("test_dir/test_file1.txt", "w") as f: f.write("Test file 1") with open("test_dir/test_file2.txt", "w") as f: f.write("Test file 2") # Call the cmd_add method with a directory commands.cmd_add("test_dir") # Check if the files have been added to the chat session self.assertIn(str(Path("test_dir/test_file1.txt").resolve()), coder.abs_fnames) self.assertIn(str(Path("test_dir/test_file2.txt").resolve()), coder.abs_fnames) >>>>>>> UPDATED ``` This new test case `test_cmd_add_directory` creates a directory named `test_dir`, adds two files to it, and attempts to add the directory using the `cmd_add` method. We then check if the files in the directory have been added to the chat session. --- tests/test_commands.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 1aa4fcba1..3a2749122 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -87,12 +87,19 @@ class TestCommands(TestCase): coder = Coder.create(models.GPT35, None, io, openai_api_key="deadbeef") commands = Commands(io, coder) - # Create a directory + # Create a directory and add files to it os.mkdir("test_dir") + with open("test_dir/test_file1.txt", "w") as f: + f.write("Test file 1") + with open("test_dir/test_file2.txt", "w") as f: + f.write("Test file 2") # Call the cmd_add method with a directory - with self.assertRaises(IsADirectoryError): - commands.cmd_add("test_dir") + commands.cmd_add("test_dir") + + # Check if the files have been added to the chat session + self.assertIn(str(Path("test_dir/test_file1.txt").resolve()), coder.abs_fnames) + self.assertIn(str(Path("test_dir/test_file2.txt").resolve()), coder.abs_fnames) def test_cmd_drop_with_glob_patterns(self): # Initialize the Commands and InputOutput objects From f48aeac88b8ed1d4aa4cd11b83e6246b44502287 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 13:26:37 -0700 Subject: [PATCH 03/10] wip --- aider/commands.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/aider/commands.py b/aider/commands.py index 43e73e87b..74c1df5c0 100644 --- a/aider/commands.py +++ b/aider/commands.py @@ -1,4 +1,5 @@ import json +import os import shlex import subprocess import sys @@ -246,14 +247,27 @@ class Commands: self.io.tool_error(f"No files to add matching pattern: {word}") else: if Path(word).exists(): - matched_files = [word] + if Path(word).is_file(): + matched_files = [word] + elif Path(word).is_dir(): + matched_files = expand_subdir(word) + else: + self.io.tool_error(f"Can not add unknown file type: {word}") elif self.io.confirm_ask( f"No files matched '{word}'. Do you want to create the file?" ): (Path(self.coder.root) / word).touch() matched_files = [word] + expanded_files = [] for matched_file in matched_files: + if Path(matched_file).is_dir(): + expanded_files += expand_subdir(matched_file) + else: + expanded_files.append(matched_file) + + dump(expanded_files) + for matched_file in expanded_files: abs_file_path = self.coder.abs_root_path(matched_file) if self.coder.repo and matched_file not in git_files: @@ -380,3 +394,9 @@ class Commands: self.io.tool_output(f"{cmd} {description}") else: self.io.tool_output(f"{cmd} No description available.") + + +def expand_subdir(file_path): + for root, dirs, files in os.walk(file_path): + for fname in files: + yield fname From 65c12a73f611a1f4e7de27515009717cada03d86 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 13:27:11 -0700 Subject: [PATCH 04/10] aider: Refactored `expand_subdir` to use `pathlib` for recursively finding all files in the given directory. # Aider chat conversation: USER: make `expand_subdir` use pathlib ASSISTANT: To modify the `expand_subdir` function to use `pathlib`, we need to modify the `aider/commands.py` file. Here are the changes: ```python aider/commands.py <<<<<<< ORIGINAL def expand_subdir(file_path): for root, dirs, files in os.walk(file_path): for fname in files: yield fname ======= from pathlib import Path def expand_subdir(file_path): for file in Path(file_path).rglob('*'): if file.is_file(): yield str(file) >>>>>>> UPDATED ``` This change replaces the `os.walk` function with the `Path.rglob` method from `pathlib`, which recursively finds all files in the given directory. The `is_file` method is used to ensure we only yield files, not directories. The `str` function is used to convert the `Path` object to a string. --- aider/commands.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/aider/commands.py b/aider/commands.py index 74c1df5c0..9d6007807 100644 --- a/aider/commands.py +++ b/aider/commands.py @@ -396,7 +396,9 @@ class Commands: self.io.tool_output(f"{cmd} No description available.") +from pathlib import Path + def expand_subdir(file_path): - for root, dirs, files in os.walk(file_path): - for fname in files: - yield fname + for file in Path(file_path).rglob('*'): + if file.is_file(): + yield str(file) From 79030d968fe3777fdcc251232d1b7c255d143778 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 13:27:43 -0700 Subject: [PATCH 05/10] use pathlib to ensure the parent dir is retained --- aider/commands.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/aider/commands.py b/aider/commands.py index 9d6007807..e2070e049 100644 --- a/aider/commands.py +++ b/aider/commands.py @@ -1,5 +1,4 @@ import json -import os import shlex import subprocess import sys @@ -266,7 +265,6 @@ class Commands: else: expanded_files.append(matched_file) - dump(expanded_files) for matched_file in expanded_files: abs_file_path = self.coder.abs_root_path(matched_file) @@ -396,9 +394,7 @@ class Commands: self.io.tool_output(f"{cmd} No description available.") -from pathlib import Path - def expand_subdir(file_path): - for file in Path(file_path).rglob('*'): + for file in Path(file_path).rglob("*"): if file.is_file(): yield str(file) From a88b2a70e82c14004eca75ee6497da58bc1d1e7d Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 15:10:23 -0700 Subject: [PATCH 06/10] cleanup --- aider/commands.py | 41 ++++++++++++++++++----------------------- tests/test_commands.py | 6 +++++- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/aider/commands.py b/aider/commands.py index e2070e049..b09fe1a5d 100644 --- a/aider/commands.py +++ b/aider/commands.py @@ -238,6 +238,7 @@ class Commands: git_added = [] git_files = self.coder.get_tracked_files() + expanded_files = set() for word in args.split(): matched_files = self.glob_filtered_to_repo(word) @@ -246,42 +247,36 @@ class Commands: self.io.tool_error(f"No files to add matching pattern: {word}") else: if Path(word).exists(): - if Path(word).is_file(): - matched_files = [word] - elif Path(word).is_dir(): - matched_files = expand_subdir(word) - else: - self.io.tool_error(f"Can not add unknown file type: {word}") + matched_files = [word] elif self.io.confirm_ask( f"No files matched '{word}'. Do you want to create the file?" ): (Path(self.coder.root) / word).touch() matched_files = [word] - expanded_files = [] for matched_file in matched_files: if Path(matched_file).is_dir(): - expanded_files += expand_subdir(matched_file) + expanded_files.update(expand_subdir(matched_file)) else: - expanded_files.append(matched_file) + expanded_files.add(matched_file) - for matched_file in expanded_files: - abs_file_path = self.coder.abs_root_path(matched_file) + for matched_file in expanded_files: + abs_file_path = self.coder.abs_root_path(matched_file) - if self.coder.repo and matched_file not in git_files: - self.coder.repo.git.add(abs_file_path) - git_added.append(matched_file) + if self.coder.repo and matched_file not in git_files: + self.coder.repo.git.add(abs_file_path) + git_added.append(matched_file) - if abs_file_path not in self.coder.abs_fnames: - content = self.io.read_text(abs_file_path) - if content is not None: - self.coder.abs_fnames.add(abs_file_path) - self.io.tool_output(f"Added {matched_file} to the chat") - added_fnames.append(matched_file) - else: - self.io.tool_error(f"Unable to read {matched_file}") + if abs_file_path not in self.coder.abs_fnames: + content = self.io.read_text(abs_file_path) + if content is not None: + self.coder.abs_fnames.add(abs_file_path) + self.io.tool_output(f"Added {matched_file} to the chat") + added_fnames.append(matched_file) else: - self.io.tool_error(f"{matched_file} is already in the chat") + self.io.tool_error(f"Unable to read {matched_file}") + else: + self.io.tool_error(f"{matched_file} is already in the chat") if self.coder.repo and git_added: git_added = " ".join(git_added) diff --git a/tests/test_commands.py b/tests/test_commands.py index 3a2749122..642b4e561 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -89,17 +89,21 @@ class TestCommands(TestCase): # Create a directory and add files to it os.mkdir("test_dir") + os.mkdir("test_dir/another_dir") with open("test_dir/test_file1.txt", "w") as f: f.write("Test file 1") with open("test_dir/test_file2.txt", "w") as f: f.write("Test file 2") + with open("test_dir/another_dir/test_file.txt", "w") as f: + f.write("Test file 3") # Call the cmd_add method with a directory - commands.cmd_add("test_dir") + commands.cmd_add("test_dir test_dir/test_file2.txt") # Check if the files have been added to the chat session self.assertIn(str(Path("test_dir/test_file1.txt").resolve()), coder.abs_fnames) self.assertIn(str(Path("test_dir/test_file2.txt").resolve()), coder.abs_fnames) + self.assertIn(str(Path("test_dir/another_dir/test_file.txt").resolve()), coder.abs_fnames) def test_cmd_drop_with_glob_patterns(self): # Initialize the Commands and InputOutput objects From 56f465b7f33425907d9f83e20a9a65e98985fb02 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 15:17:41 -0700 Subject: [PATCH 07/10] move the expansion into glob_filtered_to_repo so it works in drop too --- aider/commands.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/aider/commands.py b/aider/commands.py index b09fe1a5d..b93c28133 100644 --- a/aider/commands.py +++ b/aider/commands.py @@ -221,15 +221,18 @@ class Commands: yield Completion(fname, start_position=-len(partial)) def glob_filtered_to_repo(self, pattern): - matched_files = Path(self.coder.root).glob(pattern) - matched_files = [fn.relative_to(self.coder.root) for fn in matched_files] + raw_matched_files = Path(self.coder.root).glob(pattern) + matched_files = [] + for fn in raw_matched_files: + matched_files += expand_subdir(fn.relative_to(self.coder.root)) # if repo, filter against it if self.coder.repo: git_files = self.coder.get_tracked_files() matched_files = [fn for fn in matched_files if str(fn) in git_files] - return list(map(str, matched_files)) + res = list(map(str, matched_files)) + return res def cmd_add(self, args): "Add matching files to the chat session using glob patterns" @@ -238,7 +241,7 @@ class Commands: git_added = [] git_files = self.coder.get_tracked_files() - expanded_files = set() + all_matched_files = set() for word in args.split(): matched_files = self.glob_filtered_to_repo(word) @@ -254,13 +257,9 @@ class Commands: (Path(self.coder.root) / word).touch() matched_files = [word] - for matched_file in matched_files: - if Path(matched_file).is_dir(): - expanded_files.update(expand_subdir(matched_file)) - else: - expanded_files.add(matched_file) + all_matched_files.update(matched_files) - for matched_file in expanded_files: + for matched_file in all_matched_files: abs_file_path = self.coder.abs_root_path(matched_file) if self.coder.repo and matched_file not in git_files: @@ -390,6 +389,11 @@ class Commands: def expand_subdir(file_path): - for file in Path(file_path).rglob("*"): + file_path = Path(file_path) + if file_path.is_file(): + yield file_path + return + + for file in file_path.rglob("*"): if file.is_file(): yield str(file) From e4389ab7f7b06df53feaa2f4f96eff95c96a5d79 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 15:18:11 -0700 Subject: [PATCH 08/10] add test in drop --- tests/test_commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index 642b4e561..297fd9a30 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -105,6 +105,9 @@ class TestCommands(TestCase): self.assertIn(str(Path("test_dir/test_file2.txt").resolve()), coder.abs_fnames) self.assertIn(str(Path("test_dir/another_dir/test_file.txt").resolve()), coder.abs_fnames) + commands.cmd_drop("test_dir") + self.assertEqual(len(coder.abs_fnames), 0) + def test_cmd_drop_with_glob_patterns(self): # Initialize the Commands and InputOutput objects io = InputOutput(pretty=False, yes=True) From 93866ebe1762cbd00bd2902d6ab46072d90f4ce5 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 15:19:30 -0700 Subject: [PATCH 09/10] test name --- tests/test_commands.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 297fd9a30..3590c65e5 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -79,7 +79,7 @@ class TestCommands(TestCase): # Check if no files have been added to the chat session self.assertEqual(len(coder.abs_fnames), 0) - def test_cmd_add_directory(self): + def test_cmd_add_drop_directory(self): # Initialize the Commands and InputOutput objects io = InputOutput(pretty=False, yes=True) from aider.coders import Coder @@ -105,8 +105,12 @@ class TestCommands(TestCase): self.assertIn(str(Path("test_dir/test_file2.txt").resolve()), coder.abs_fnames) self.assertIn(str(Path("test_dir/another_dir/test_file.txt").resolve()), coder.abs_fnames) - commands.cmd_drop("test_dir") - self.assertEqual(len(coder.abs_fnames), 0) + commands.cmd_drop("test_dir/another_dir") + self.assertIn(str(Path("test_dir/test_file1.txt").resolve()), coder.abs_fnames) + self.assertIn(str(Path("test_dir/test_file2.txt").resolve()), coder.abs_fnames) + self.assertNotIn( + str(Path("test_dir/another_dir/test_file.txt").resolve()), coder.abs_fnames + ) def test_cmd_drop_with_glob_patterns(self): # Initialize the Commands and InputOutput objects From baa11b0c0d67444deeefdf1e49a08df3bdb8fa02 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 10 Jul 2023 15:23:25 -0700 Subject: [PATCH 10/10] cleanup --- aider/commands.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/aider/commands.py b/aider/commands.py index b93c28133..396c960b0 100644 --- a/aider/commands.py +++ b/aider/commands.py @@ -266,16 +266,16 @@ class Commands: self.coder.repo.git.add(abs_file_path) git_added.append(matched_file) - if abs_file_path not in self.coder.abs_fnames: + if abs_file_path in self.coder.abs_fnames: + self.io.tool_error(f"{matched_file} is already in the chat") + else: content = self.io.read_text(abs_file_path) - if content is not None: + if content is None: + self.io.tool_error(f"Unable to read {matched_file}") + else: self.coder.abs_fnames.add(abs_file_path) self.io.tool_output(f"Added {matched_file} to the chat") added_fnames.append(matched_file) - else: - self.io.tool_error(f"Unable to read {matched_file}") - else: - self.io.tool_error(f"{matched_file} is already in the chat") if self.coder.repo and git_added: git_added = " ".join(git_added)