Merge branch 'main' into sitter-map

aider/__init__.py

@@ -1 +1 @@
-__version__ = "0.13.1-dev"
+__version__ = "0.14.3-dev"

aider/coders/base_coder.py

@@ -55,6 +55,7 @@ class Coder:
         main_model,
         edit_format,
         io,
+        skip_model_availabily_check=False,
         **kwargs,
     ):
         from . import EditBlockCoder, WholeFileCoder
@@ -62,8 +63,8 @@ class Coder:
         if not main_model:
             main_model = models.GPT35_16k
 
-        if not main_model.always_available:
-            if not check_model_availability(main_model):
+        if not skip_model_availabily_check and not main_model.always_available:
+            if not check_model_availability(io, main_model):
                 if main_model != models.GPT4:
                     io.tool_error(
                         f"API key does not support {main_model.name}, falling back to"
@@ -99,6 +100,7 @@ class Coder:
         stream=True,
         use_git=True,
         voice_language=None,
+        aider_ignore_file=None,
     ):
         if not fnames:
             fnames = []
@@ -153,7 +155,7 @@ class Coder:
 
         if use_git:
             try:
-                self.repo = GitRepo(self.io, fnames, git_dname)
+                self.repo = GitRepo(self.io, fnames, git_dname, aider_ignore_file)
                 self.root = self.repo.root
             except FileNotFoundError:
                 self.repo = None
@@ -185,7 +187,7 @@ class Coder:
 
         self.summarizer = ChatSummary(models.Model.weak_model())
         self.summarizer_thread = None
-        self.summarized_done_messages = None
+        self.summarized_done_messages = []
 
         # validate the functions jsonschema
         if self.functions:
@@ -352,7 +354,11 @@ class Coder:
         self.summarizer_thread.start()
 
     def summarize_worker(self):
-        self.summarized_done_messages = self.summarizer.summarize(self.done_messages)
+        try:
+            self.summarized_done_messages = self.summarizer.summarize(self.done_messages)
+        except ValueError as err:
+            self.io.tool_error(err.args[0])
+
         if self.verbose:
             self.io.tool_output("Finished summarizing chat history.")
 
@@ -364,7 +370,7 @@ class Coder:
         self.summarizer_thread = None
 
         self.done_messages = self.summarized_done_messages
-        self.summarized_done_messages = None
+        self.summarized_done_messages = []
 
     def move_back_cur_messages(self, message):
         self.done_messages += self.cur_messages
@@ -595,15 +601,17 @@ class Coder:
             self.io.tool_error(show_content_err)
             raise Exception("No data found in openai response!")
 
-        prompt_tokens = completion.usage.prompt_tokens
-        completion_tokens = completion.usage.completion_tokens
+        tokens = None
+        if hasattr(completion, "usage"):
+            prompt_tokens = completion.usage.prompt_tokens
+            completion_tokens = completion.usage.completion_tokens
 
-        tokens = f"{prompt_tokens} prompt tokens, {completion_tokens} completion tokens"
-        if self.main_model.prompt_price:
-            cost = prompt_tokens * self.main_model.prompt_price / 1000
-            cost += completion_tokens * self.main_model.completion_price / 1000
-            tokens += f", ${cost:.6f} cost"
-            self.total_cost += cost
+            tokens = f"{prompt_tokens} prompt tokens, {completion_tokens} completion tokens"
+            if self.main_model.prompt_price:
+                cost = prompt_tokens * self.main_model.prompt_price / 1000
+                cost += completion_tokens * self.main_model.completion_price / 1000
+                tokens += f", ${cost:.6f} cost"
+                self.total_cost += cost
 
         show_resp = self.render_incremental_response(True)
         if self.pretty:
@@ -614,7 +622,9 @@ class Coder:
             show_resp = Text(show_resp or "<no response>")
 
         self.io.console.print(show_resp)
-        self.io.tool_output(tokens)
+
+        if tokens is not None:
+            self.io.tool_output(tokens)
 
     def show_send_output_stream(self, completion):
         live = None
@@ -895,7 +905,12 @@ class Coder:
         return True
 
 
-def check_model_availability(main_model):
+def check_model_availability(io, main_model):
     available_models = openai.Model.list()
     model_ids = [model.id for model in available_models["data"]]
-    return main_model.name in model_ids
+    if main_model.name in model_ids:
+        return True
+
+    available_models = ", ".join(model_ids)
+    io.tool_error(f"API key supports: {available_models}")
+    return False

aider/commands.py

@@ -1,5 +1,5 @@
 import json
-import shlex
+import re
 import subprocess
 import sys
 from pathlib import Path
@@ -209,6 +209,10 @@ class Commands:
             or last_commit.hexsha[:7] != self.coder.last_aider_commit_hash
         ):
             self.io.tool_error("The last commit was not made by aider in this chat session.")
+            self.io.tool_error(
+                "You could try `/git reset --hard HEAD^` but be aware that this is a destructive"
+                " command!"
+            )
             return
         self.coder.repo.repo.git.reset("--hard", "HEAD~1")
         self.io.tool_output(
@@ -228,6 +232,7 @@ class Commands:
 
         if not self.coder.last_aider_commit_hash:
             self.io.tool_error("No previous aider commit found.")
+            self.io.tool_error("You could try `/git diff` or `/git diff HEAD^`.")
             return
 
         commits = f"{self.coder.last_aider_commit_hash}~1"
@@ -272,29 +277,37 @@ class Commands:
         git_files = self.coder.repo.get_tracked_files() if self.coder.repo else []
 
         all_matched_files = set()
-        for word in args.split():
+
+        filenames = parse_quoted_filenames(args)
+        for word in filenames:
+            if Path(word).is_absolute():
+                fname = Path(word)
+            else:
+                fname = Path(self.coder.root) / word
+
+            if fname.exists() and fname.is_file():
+                all_matched_files.add(str(fname))
+                continue
+                # an existing dir will fall through and get recursed by glob
+
             matched_files = self.glob_filtered_to_repo(word)
+            if matched_files:
+                all_matched_files.update(matched_files)
+                continue
 
-            if not matched_files:
-                if any(char in word for char in "*?[]"):
-                    self.io.tool_error(f"No files to add matching pattern: {word}")
-                else:
-                    if Path(word).exists():
-                        if Path(word).is_file():
-                            matched_files = [word]
-                        else:
-                            self.io.tool_error(f"Unable to add: {word}")
-                    elif self.io.confirm_ask(
-                        f"No files matched '{word}'. Do you want to create the file?"
-                    ):
-                        (Path(self.coder.root) / word).touch()
-                        matched_files = [word]
-
-            all_matched_files.update(matched_files)
+            if self.io.confirm_ask(f"No files matched '{word}'. Do you want to create {fname}?"):
+                fname.touch()
+                all_matched_files.add(str(fname))
 
         for matched_file in all_matched_files:
             abs_file_path = self.coder.abs_root_path(matched_file)
 
+            if not abs_file_path.startswith(self.coder.root):
+                self.io.tool_error(
+                    f"Can not add {abs_file_path}, which is not within {self.coder.root}"
+                )
+                continue
+
             if self.coder.repo and matched_file not in git_files:
                 self.coder.repo.repo.git.add(abs_file_path)
                 git_added.append(matched_file)
@@ -339,7 +352,8 @@ class Commands:
             self.io.tool_output("Dropping all files from the chat session.")
             self.coder.abs_fnames = set()
 
-        for word in args.split():
+        filenames = parse_quoted_filenames(args)
+        for word in filenames:
             matched_files = self.glob_filtered_to_repo(word)
 
             if not matched_files:
@@ -355,10 +369,15 @@ class Commands:
         "Run a git command"
         combined_output = None
         try:
-            parsed_args = shlex.split("git " + args)
+            args = "git " + args
             env = dict(GIT_EDITOR="true", **subprocess.os.environ)
             result = subprocess.run(
-                parsed_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env
+                args,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                env=env,
+                shell=True,
             )
             combined_output = result.stdout
         except Exception as e:
@@ -373,9 +392,8 @@ class Commands:
         "Run a shell command and optionally add the output to the chat"
         combined_output = None
         try:
-            parsed_args = shlex.split(args)
             result = subprocess.run(
-                parsed_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+                args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, shell=True
             )
             combined_output = result.stdout
         except Exception as e:
@@ -447,7 +465,9 @@ class Commands:
             try:
                 self.voice = voice.Voice()
             except voice.SoundDeviceError:
-                self.io.tool_error("Unable to import `sounddevice`, is portaudio installed?")
+                self.io.tool_error(
+                    "Unable to import `sounddevice` and/or `soundfile`, is portaudio installed?"
+                )
                 return
 
         history_iter = self.io.get_input_history()
@@ -487,3 +507,9 @@ def expand_subdir(file_path):
         for file in file_path.rglob("*"):
             if file.is_file():
                 yield str(file)
+
+
+def parse_quoted_filenames(args):
+    filenames = re.findall(r"\"(.+?)\"|(\S+)", args)
+    filenames = [name for sublist in filenames for name in sublist if name]
+    return filenames

aider/history.py

@@ -85,6 +85,8 @@ class ChatSummary:
         ]
 
         summary = simple_send_with_retries(self.model.name, messages)
+        if summary is None:
+            raise ValueError(f"summarizer unexpectedly failed for {self.model.name}")
         summary = prompts.summary_prefix + summary
 
         return [dict(role="user", content=summary)]

aider/main.py

@@ -60,10 +60,10 @@ def setup_git(git_root, io):
     with repo.config_writer() as git_config:
         if not global_git_config.has_option("user", "name"):
             git_config.set_value("user", "name", "Your Name")
-            io.tool_error('Update git name with: git config --global user.name "Your Name"')
+            io.tool_error('Update git name with: git config user.name "Your Name"')
         if not global_git_config.has_option("user", "email"):
             git_config.set_value("user", "email", "you@example.com")
-            io.tool_error('Update git email with: git config --global user.email "you@example.com"')
+            io.tool_error('Update git email with: git config user.email "you@example.com"')
 
     io.tool_output("Git repository created in the current working directory.")
 
@@ -74,6 +74,13 @@ def check_gitignore(git_root, io, ask=True):
     if not git_root:
         return
 
+    try:
+        repo = git.Repo(git_root)
+        if repo.ignored(".aider"):
+            return
+    except git.exc.InvalidGitRepositoryError:
+        pass
+
     pat = ".aider*"
 
     gitignore_file = Path(git_root) / ".gitignore"
@@ -142,6 +149,12 @@ def main(argv=None, input=None, output=None, force_git_root=None):
         default=models.GPT4.name,
         help=f"Specify the model to use for the main chat (default: {models.GPT4.name})",
     )
+    core_group.add_argument(
+        "--skip-model-availability-check",
+        metavar="SKIP_MODEL_AVAILABILITY_CHECK",
+        default=False,
+        help="Override to skip model availability check (default: False)",
+    )
     core_group.add_argument(
         "-3",
         action="store_const",
@@ -286,6 +299,21 @@ def main(argv=None, input=None, output=None, force_git_root=None):
         default=True,
         help="Enable/disable looking for a git repo (default: True)",
     )
+    git_group.add_argument(
+        "--gitignore",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Enable/disable adding .aider* to .gitignore (default: True)",
+    )
+    default_aiderignore_file = (
+        os.path.join(git_root, ".aiderignore") if git_root else ".aiderignore"
+    )
+    git_group.add_argument(
+        "--aiderignore",
+        metavar="AIDERIGNORE",
+        default=default_aiderignore_file,
+        help="Specify the aider ignore file (default: .aiderignore in git root)",
+    )
     git_group.add_argument(
         "--auto-commits",
         action=argparse.BooleanOptionalAction,
@@ -302,7 +330,13 @@ def main(argv=None, input=None, output=None, force_git_root=None):
         "--dry-run",
         action=argparse.BooleanOptionalAction,
         default=False,
-        help="Enable/disable performing a dry run without modifying files (default: False)",
+        help="Perform a dry run without modifying files (default: False)",
+    )
+    git_group.add_argument(
+        "--commit",
+        action="store_true",
+        help="Commit all pending changes with a suitable commit message, then exit",
+        default=False,
     )
 
     ##########
@@ -423,7 +457,8 @@ def main(argv=None, input=None, output=None, force_git_root=None):
 
     if args.git:
         git_root = setup_git(git_root, io)
-        check_gitignore(git_root, io)
+        if args.gitignore:
+            check_gitignore(git_root, io)
 
     def scrub_sensitive_info(text):
         # Replace sensitive information with placeholder
@@ -465,6 +500,7 @@ def main(argv=None, input=None, output=None, force_git_root=None):
             main_model,
             args.edit_format,
             io,
+            args.skip_model_availability_check,
             ##
             fnames=fnames,
             git_dname=git_dname,
@@ -480,11 +516,16 @@ def main(argv=None, input=None, output=None, force_git_root=None):
             stream=args.stream,
             use_git=args.git,
             voice_language=args.voice_language,
+            aider_ignore_file=args.aiderignore,
         )
     except ValueError as err:
         io.tool_error(str(err))
         return 1
 
+    if args.commit:
+        coder.commands.cmd_commit("")
+        return
+
     if args.show_repo_map:
         repo_map = coder.get_repo_map()
         if repo_map:
@@ -501,6 +542,15 @@ def main(argv=None, input=None, output=None, force_git_root=None):
 
     io.tool_output("Use /help to see in-chat commands, run with --help to see cmd line args")
 
+    if git_root and Path.cwd().resolve() != Path(git_root).resolve():
+        io.tool_error(
+            "Note: in-chat filenames are always relative to the git working dir, not the current"
+            " working dir."
+        )
+
+        io.tool_error(f"Cur working dir: {Path.cwd()}")
+        io.tool_error(f"Git working dir: {git_root}")
+
     if args.message:
         io.tool_output()
         coder.run(with_message=args.message)

aider/models/__init__.py

@@ -1,5 +1,15 @@
 from .model import Model
+from .openai import OpenAIModel
+from .openrouter import OpenRouterModel
 
 GPT4 = Model.create("gpt-4")
 GPT35 = Model.create("gpt-3.5-turbo")
 GPT35_16k = Model.create("gpt-3.5-turbo-16k")
+
+__all__ = [
+    OpenAIModel,
+    OpenRouterModel,
+    GPT4,
+    GPT35,
+    GPT35_16k,
+]

aider/repo.py

@@ -2,6 +2,7 @@ import os
 from pathlib import Path, PurePosixPath
 
 import git
+import pathspec
 
 from aider import models, prompts, utils
 from aider.sendchat import simple_send_with_retries
@@ -11,8 +12,11 @@ from .dump import dump  # noqa: F401
 
 class GitRepo:
     repo = None
+    aider_ignore_file = None
+    aider_ignore_spec = None
+    aider_ignore_ts = 0
 
-    def __init__(self, io, fnames, git_dname):
+    def __init__(self, io, fnames, git_dname, aider_ignore_file=None):
         self.io = io
 
         if git_dname:
@@ -49,14 +53,20 @@ class GitRepo:
         self.repo = git.Repo(repo_paths.pop(), odbt=git.GitDB)
         self.root = utils.safe_abs_path(self.repo.working_tree_dir)
 
+        if aider_ignore_file:
+            self.aider_ignore_file = Path(aider_ignore_file)
+
     def commit(self, fnames=None, context=None, prefix=None, message=None):
         if not fnames and not self.repo.is_dirty():
             return
 
+        diffs = self.get_diffs(fnames)
+        if not diffs:
+            return
+
         if message:
             commit_message = message
         else:
-            diffs = self.get_diffs(fnames)
             commit_message = self.get_commit_message(diffs, context)
 
         if not commit_message:
@@ -190,7 +200,22 @@ class GitRepo:
             for path in files
         )
 
-        return res
+        return self.filter_ignored_files(res)
+
+    def filter_ignored_files(self, fnames):
+        if not self.aider_ignore_file or not self.aider_ignore_file.is_file():
+            return fnames
+
+        mtime = self.aider_ignore_file.stat().st_mtime
+        if mtime != self.aider_ignore_ts:
+            self.aider_ignore_ts = mtime
+            lines = self.aider_ignore_file.read_text().splitlines()
+            self.aider_ignore_spec = pathspec.PathSpec.from_lines(
+                pathspec.patterns.GitWildMatchPattern,
+                lines,
+            )
+
+        return [fname for fname in fnames if not self.aider_ignore_spec.match_file(fname)]
 
     def path_in_repo(self, path):
         if not self.repo:

aider/repomap.py

@@ -83,6 +83,8 @@ class RepoMap:
 
     cache_missing = False
 
+    warned_files = set()
+
     def __init__(
         self,
         map_tokens=1024,
@@ -235,7 +237,10 @@ class RepoMap:
 
         for fname in fnames:
             if not Path(fname).is_file():
-                self.io.tool_error(f"Repo-map can't include {fname}")
+                if fname not in self.warned_files:
+                    self.io.tool_error(f"Repo-map can't include {fname}")
+
+                self.warned_files.add(fname)
                 continue
 
             # dump(fname)

aider/voice.py

@@ -5,7 +5,12 @@ import time
 
 import numpy as np
 import openai
-import soundfile as sf
+
+try:
+    import soundfile as sf
+except (OSError, ModuleNotFoundError):
+    sf = None
+
 from prompt_toolkit.shortcuts import prompt
 
 from .dump import dump  # noqa: F401
@@ -23,12 +28,14 @@ class Voice:
     threshold = 0.15
 
     def __init__(self):
+        if sf is None:
+            raise SoundDeviceError
         try:
             print("Initializing sound device...")
             import sounddevice as sd
 
             self.sd = sd
-        except OSError:
+        except (OSError, ModuleNotFoundError):
             raise SoundDeviceError
 
     def callback(self, indata, frames, time, status):