moved into subdir

This commit is contained in:
Paul Gauthier 2023-05-09 10:52:18 -07:00
parent a6b9023079
commit 984e2f78ce
7 changed files with 0 additions and 0 deletions

442
aider/coder.py Executable file
View file

@ -0,0 +1,442 @@
#!/usr/bin/env python
import sys
import re
import traceback
from rich.console import Console
from rich.prompt import Confirm, Prompt
from rich.live import Live
from rich.text import Text
from rich.markdown import Markdown
from pathlib import Path
from getinput import get_input
import utils
import os
import git
import openai
from dump import dump
import prompts
openai.api_key = os.getenv("OPENAI_API_KEY")
class Coder:
fnames = set()
last_modified = 0
repo = None
def __init__(self, main_model, files, pretty, history_file=".coder.history"):
self.history_file = history_file
if pretty:
self.console = Console()
else:
self.console = Console(force_terminal=True, no_color=True)
self.main_model = main_model
if main_model == "gpt-3.5-turbo":
self.console.print(
f"[red bold]This tool will almost certainly fail to work with {main_model}"
)
for fname in files:
fname = Path(fname)
if not fname.exists():
self.console.print(f"[red]Creating {fname}")
fname.touch()
else:
self.console.print(f"[red]Loading {fname}")
self.fnames.add(str(fname))
self.set_repo()
if not self.repo:
self.console.print(
"[red bold]No suitable git repo, will not automatically commit edits."
)
self.pretty = pretty
def set_repo(self):
repo_paths = list(
git.Repo(fname, search_parent_directories=True).git_dir
for fname in self.fnames
)
num_repos = len(set(repo_paths))
if num_repos == 0:
self.console.print("[red bold]Files are not in a git repo.")
return
if num_repos > 1:
self.console.print("[red bold]Files are in different git repos.")
return
repo = git.Repo(repo_paths.pop())
new_files = []
for fname in self.fnames:
relative_fname = os.path.relpath(fname, repo.working_tree_dir)
tracked_files = set(repo.git.ls_files().splitlines())
if relative_fname not in tracked_files:
new_files.append(relative_fname)
if new_files:
self.console.print(f"[red bold]Files not tracked in {repo.git_dir}:")
for fn in new_files:
self.console.print(f"[red bold] {fn}")
if Confirm.ask("[bold red]Add them?", console=self.console):
for relative_fname in new_files:
repo.git.add(relative_fname)
self.console.print(
f"[red bold]Added {relative_fname} to the git repo"
)
show_files = ", ".join(new_files)
commit_message = (
f"Initial commit: Added new files to the git repo: {show_files}"
)
repo.git.commit("-m", commit_message, "--no-verify")
self.console.print(
f"[green bold]Committed new files with message: {commit_message}"
)
else:
self.console.print(
"[red bold]Skipped adding new files to the git repo."
)
return
self.repo = repo
def get_files_content(self):
prompt = ""
for fname in self.fnames:
prompt += utils.quoted_file(fname)
return prompt
def get_last_modified(self):
return max(Path(fname).stat().st_mtime for fname in self.fnames)
def get_files_messages(self):
files_content = prompts.files_content_prefix
files_content += self.get_files_content()
files_messages = [
dict(role="user", content=files_content),
dict(role="assistant", content="Ok."),
dict(
role="system",
content=prompts.files_content_suffix + prompts.system_reminder,
),
]
return files_messages
def run(self):
self.done_messages = []
self.cur_messages = []
self.num_control_c = 0
while True:
try:
self.run_loop()
except KeyboardInterrupt:
self.num_control_c += 1
if self.num_control_c >= 2:
break
self.console.print("[bold red]^C again to quit")
def run_loop(self):
if self.pretty:
self.console.rule()
else:
print()
inp = get_input(self.history_file, self.fnames)
if inp is None:
return
self.num_control_c = 0
if self.last_modified < self.get_last_modified():
self.commit(ask=True)
# files changed, move cur messages back behind the files messages
self.done_messages += self.cur_messages
self.done_messages += [
dict(role="user", content=prompts.files_content_local_edits),
dict(role="assistant", content="Ok."),
]
self.cur_messages = []
self.cur_messages += [
dict(role="user", content=inp),
]
messages = [
dict(role="system", content=prompts.main_system + prompts.system_reminder),
]
messages += self.done_messages
messages += self.get_files_messages()
messages += self.cur_messages
# self.show_messages(messages, "all")
content, interrupted = self.send(messages)
if interrupted:
content += "\n^C KeyboardInterrupt"
Path("tmp.last-edit.md").write_text(content)
self.cur_messages += [
dict(role="assistant", content=content),
]
self.console.print()
if interrupted:
return True
try:
edited = self.update_files(content, inp)
except Exception as err:
print(err)
print()
traceback.print_exc()
edited = None
if not edited:
return True
res = self.commit(history=self.cur_messages)
if res:
commit_hash, commit_message = res
saved_message = prompts.files_content_gpt_edits.format(
hash=commit_hash,
message=commit_message,
)
else:
self.console.print("[red bold]No changes found in tracked files.")
saved_message = prompts.files_content_gpt_no_edits
self.done_messages += self.cur_messages
self.done_messages += [
dict(role="user", content=saved_message),
dict(role="assistant", content="Ok."),
]
self.cur_messages = []
return True
def show_messages(self, messages, title):
print(title.upper(), "*" * 50)
for msg in messages:
print()
print("-" * 50)
role = msg["role"].upper()
content = msg["content"].splitlines()
for line in content:
print(role, line)
def send(self, messages, model=None, silent=False):
# self.show_messages(messages, "all")
if not model:
model = self.main_model
import time
from openai.error import RateLimitError
self.resp = ""
interrupted = False
try:
while True:
try:
completion = openai.ChatCompletion.create(
model=model,
messages=messages,
temperature=0,
stream=True,
)
break
except RateLimitError as e:
retry_after = e.retry_after
print(f"Rate limit exceeded. Retrying in {retry_after} seconds.")
time.sleep(retry_after)
if self.pretty and not silent:
self.show_send_output_color(completion)
else:
self.show_send_output_plain(completion, silent)
except KeyboardInterrupt:
interrupted = True
return self.resp, interrupted
def show_send_output_plain(self, completion, silent):
for chunk in completion:
if chunk.choices[0].finish_reason not in (None, "stop"):
dump(chunk.choices[0].finish_reason)
try:
text = chunk.choices[0].delta.content
self.resp += text
except AttributeError:
continue
if not silent:
sys.stdout.write(text)
sys.stdout.flush()
def show_send_output_color(self, completion):
with Live(vertical_overflow="scroll") as live:
for chunk in completion:
if chunk.choices[0].finish_reason not in (None, "stop"):
assert False, "Exceeded context window!"
try:
text = chunk.choices[0].delta.content
self.resp += text
except AttributeError:
continue
md = Markdown(self.resp, style="blue", code_theme="default")
live.update(md)
live.update(Text(""))
live.stop()
md = Markdown(self.resp, style="blue", code_theme="default")
self.console.print(md)
pattern = re.compile(
r"(^```\S*\s*)?^((?:[a-zA-Z]:\\|/)?(?:[\w\s.-]+[\\/])*\w+(\.[\w\s.-]+)*)\s+(^```\S*\s*)?^<<<<<<< ORIGINAL\n(.*?\n?)^=======\n(.*?)^>>>>>>> UPDATED", # noqa: E501
re.MULTILINE | re.DOTALL,
)
def update_files(self, content, inp):
edited = set()
for match in self.pattern.finditer(content):
_, path, _, _, original, updated = match.groups()
if path not in self.fnames:
if not Path(path).exists():
question = f"[red bold]Allow creation of new file {path}?"
else:
question = f"[red bold]Allow edits to {path} which was not previously provided?"
if not Confirm.ask(question, console=self.console):
self.console.print(f"[red]Skipping edit to {path}")
continue
self.fnames.add(path)
edited.add(path)
if utils.do_replace(path, original, updated):
self.console.print(f"[red]Applied edit to {path}")
else:
self.console.print(f"[red]Failed to apply edit to {path}")
return edited
def commit(self, history=None, prefix=None, ask=False):
repo = self.repo
if not repo:
return
if not repo.is_dirty():
return
diffs = ""
dirty_fnames = []
relative_dirty_fnames = []
for fname in self.fnames:
relative_fname = os.path.relpath(fname, repo.working_tree_dir)
if self.pretty:
these_diffs = repo.git.diff("HEAD", "--color", relative_fname)
else:
these_diffs = repo.git.diff("HEAD", relative_fname)
if these_diffs:
dirty_fnames.append(fname)
relative_dirty_fnames.append(relative_fname)
diffs += these_diffs + "\n"
if not dirty_fnames:
self.last_modified = self.get_last_modified()
return
self.console.print(Text(diffs))
diffs = "# Diffs:\n" + diffs
# for fname in dirty_fnames:
# self.console.print(f"[red] {fname}")
context = ""
if history:
context += "# Context:\n"
for msg in history:
context += msg["role"].upper() + ": " + msg["content"] + "\n"
messages = [
dict(role="system", content=prompts.commit_system),
dict(role="user", content=context + diffs),
]
# if history:
# self.show_messages(messages, "commit")
commit_message, interrupted = self.send(
messages,
model="gpt-3.5-turbo",
silent=True,
)
commit_message = commit_message.strip().strip('"').strip()
if interrupted:
commit_message = "Saving dirty files before chat"
if prefix:
commit_message = prefix + commit_message
if ask:
self.last_modified = self.get_last_modified()
self.console.print("[red]Files have uncommitted changes.\n")
self.console.print(f"[red]Suggested commit message:\n{commit_message}\n")
res = Prompt.ask(
"[red]Commit before the chat proceeds? \[Y/n/commit message]", # noqa: W605
console=self.console,
).strip()
self.console.print()
if res.lower() in ["n", "no"]:
self.console.print("[red]Skipped commmit.")
return
if res.lower() not in ["y", "yes"] and res:
commit_message = res
repo.git.add(*relative_dirty_fnames)
full_commit_message = commit_message + "\n\n" + context
repo.git.commit("-m", full_commit_message, "--no-verify")
commit_hash = repo.head.commit.hexsha[:7]
self.console.print(f"[green]{commit_hash} {commit_message}")
self.last_modified = self.get_last_modified()
return commit_hash, commit_message
if __name__ == "__main__":
from main import main
status = main()
sys.exit(status)

27
aider/dump.py Normal file
View file

@ -0,0 +1,27 @@
import traceback
import json
def cvt(s):
if isinstance(s, str):
return s
try:
return json.dumps(s, indent = 4)
except TypeError:
return str(s)
def dump(*vals):
# http://docs.python.org/library/traceback.html
stack= traceback.extract_stack()
vars= stack[-2][3]
# strip away the call to dump()
vars= '('.join(vars.split('(')[1:])
vars= ')'.join(vars.split(')')[:-1])
vals= [cvt(v) for v in vals]
has_newline = sum(1 for v in vals if '\n' in v)
if has_newline:
print('%s:' % vars)
print(', '.join(vals))
else:
print('%s:' % vars, ', '.join(vals))

61
aider/getinput.py Normal file
View file

@ -0,0 +1,61 @@
from prompt_toolkit.styles import Style
from prompt_toolkit import prompt
from prompt_toolkit.completion import Completer, Completion
from prompt_toolkit.history import FileHistory
class FileContentCompleter(Completer):
def __init__(self, fnames):
self.fnames = fnames
def get_completions(self, document, complete_event):
text = document.text_before_cursor
words = text.split()
if not words:
return
last_word = words[-1]
for fname in self.fnames:
with open(fname, "r") as f:
content = f.read()
for word in content.split():
if word.startswith(last_word):
yield Completion(word, start_position=-len(last_word))
def get_input(history_file, fnames):
inp = ""
multiline_input = False
style = Style.from_dict({"": "green"})
while True:
completer_instance = FileContentCompleter(fnames)
if multiline_input:
show = ". "
else:
show = "> "
try:
line = prompt(
show,
completer=completer_instance,
history=FileHistory(history_file),
style=style,
)
except EOFError:
return
if line.strip() == "{" and not multiline_input:
multiline_input = True
continue
elif line.strip() == "}" and multiline_input:
break
elif multiline_input:
inp += line + "\n"
else:
inp = line
break
print()
return inp

76
aider/main.py Normal file
View file

@ -0,0 +1,76 @@
import os
import sys
import argparse
from dotenv import load_dotenv
from coder import Coder
def main():
load_dotenv()
env_prefix = "AIDER_"
parser = argparse.ArgumentParser(
description="aider - chat with GPT about your code"
)
parser.add_argument(
"files",
metavar="FILE",
nargs="+",
help="a list of source code files",
)
parser.add_argument(
"--history-file",
metavar="HISTORY_FILE",
default=os.environ.get(f"{env_prefix}HISTORY_FILE", ".aider.history"),
help=f"Specify the history file (default: .aider.history, ${env_prefix}HISTORY_FILE)",
)
parser.add_argument(
"--model",
metavar="MODEL",
default="gpt-4",
help="Specify the model to use for the main chat (default: gpt-4)",
)
parser.add_argument(
"-3",
action="store_const",
dest="model",
const="gpt-3.5-turbo",
help="Use gpt-3.5-turbo model for the main chat (basically won't work)",
)
parser.add_argument(
"--no-pretty",
action="store_false",
dest="pretty",
help=f"Disable pretty, colorized output (${env_prefix}PRETTY)",
default=bool(int(os.environ.get(f"{env_prefix}PRETTY", 1))),
)
parser.add_argument(
"--apply",
metavar="FILE",
help="Apply the changes from the given file instead of running the chat",
)
parser.add_argument(
"--commit-dirty",
action="store_true",
help=f"On launch, commit dirty files w/o confirmation (default: False, ${env_prefix}COMMIT_DIRTY)", # noqa: E501
default=bool(int(os.environ.get(f"{env_prefix}COMMIT_DIRTY", 0))),
)
args = parser.parse_args()
fnames = args.files
pretty = args.pretty
coder = Coder(args.model, fnames, pretty, args.history_file)
coder.commit(ask=not args.commit_dirty, prefix="WIP: ")
if args.apply:
with open(args.apply, "r") as f:
content = f.read()
coder.update_files(content, inp="")
return
coder.run()
if __name__ == "__main__":
status = main()
sys.exit(status)

124
aider/prompts.py Normal file
View file

@ -0,0 +1,124 @@
# flake8: noqa: E501
# MAIN
main_system = """
I want you to act as an expert software engineer and pair programmer.
The user will show you the files in the following triple-quoted format.
NEVER REPLY USING THIS FORMAT!
some/dir/example.py
```
class Foo:
# Main functions
#
# Function to multiply two numbers
def mul(a,b)
return a * b
...
```
Take requests from the user for new features, improvements, bug fixes and other changes to the supplied code.
If the user's request is ambiguous, ask questions to fully understand.
Once you understand the user's request, your responses MUST be:
1. Briefly explain the needed changes.
2. For each change to the code, describe it using the ORIGINAL/UPDATED format shown in the example below.
"""
system_reminder = '''
You must format every code change like this example:
```python
some/dir/example.py
<<<<<<< ORIGINAL
# Main functions
#
# Function to multiply two numbers
def mul(a,b)
=======
# Main functions are below.
# Add new ones in this section
# Function to multiply two numbers using the standard algorithm
def mul(a,b):
"""Multiplies 2 numbers"""
>>>>>>> UPDATED
*NEVER REPLY WITH AN ENTIRE FILE TRIPLE-QUOTED FORMAT LIKE THE USER MESSAGES!*
*ANY CODE YOU INCLUDE IN A REPLY *MUST* BE IN THE ORIGINAL/UPDATED FORMAT!*
EVERY ORIGINAL/UPDATED BLOCK MUST START WITH THE FILENAME!
EVERY ORIGINAL/UPDATED BLOCK MUST BE TRIPLE QUOTED!
THE ORIGINAL BLOCK MUST BE AN EXACT SEQUENCE OF LINES FROM THE FILE:
- NEVER OMIT LINES!
- INCLUDE ALL THE ORIGINAL LEADING SPACES AND INDENTATION!
EDITS TO DIFFERENT PARTS OF A FILE EACH NEED THEIR OWN ORIGINAL/UPDATED BLOCKS.
EVEN NEARBY PARTS NEED THEIR OWN ORIGINAL/UPDATED BLOCKS.
INCLUDE THE FILE PATH ALONE AS THE FIRST LINE OF THE BLOCK.
Don't prefix it with "In" or follow it with ":".
'''
returned_code = """
It looks like you tried to return a code block. Don't do that!
Only return code using the specific ORIGINAL/UPDATED format.
Be selective!
Only return the parts of the code which need changes!
"""
# FILES
files_content_gpt_edits = "I committed your suggested changes with git hash {hash} and commit message: {message}"
files_content_gpt_no_edits = (
"I wasn't able to see any properly formatted edits in your reply?!"
)
files_content_local_edits = "I made some changes to the files myself."
files_content_prefix = "Here is the current content of the files:\n\n"
files_content_suffix = """Base any edits on the current contents of the files as shown in the user's last message."""
# EDITOR
editor_system = """
You are an expert code editor.
Perform the requested edit.
Output ONLY the new version of the file.
Just that one file.
Do not output explanations!
Do not wrap the output in ``` delimiters.
"""
editor_user = """
To complete this request:
{request}
You need to apply this change:
{edit}
To this file:
{fname}
```
{content}
```
ONLY OUTPUT {fname} !!!
"""
# COMMIT
commit_system = """You are an expert software engineer.
Review the provided context and diffs which are about to be committed to a git repo.
Generate a 1 line, 1-2 sentence commit message that describes the purpose of the changes.
Reply with JUST the commit message, without quotes, comments, questions, etc.
"""

42
aider/test_utils.py Normal file
View file

@ -0,0 +1,42 @@
import unittest
from utils import replace_most_similar_chunk
class TestUtils(unittest.TestCase):
def test_replace_most_similar_chunk(self):
whole = "This is a sample text.\nAnother line of text.\nYet another line."
part = "This is a sample text"
replace = "This is a replaced text."
expected_output = "This is a replaced text.\nAnother line of text.\nYet another line."
result = replace_most_similar_chunk(whole, part, replace)
self.assertEqual(result, expected_output)
def test_replace_most_similar_chunk_not_perfect_match(self):
whole = "This is a sample text.\nAnother line of text.\nYet another line."
part = "This was a sample text.\nAnother line of txt"
replace = "This is a replaced text.\nModified line of text."
expected_output = "This is a replaced text.\nModified line of text.\nYet another line."
result = replace_most_similar_chunk(whole, part, replace)
self.assertEqual(result, expected_output)
def test_strip_quoted_wrapping(self):
input_text = "filename.ext\n```\nWe just want this content\nNot the filename and triple quotes\n```"
expected_output = "We just want this content\nNot the filename and triple quotes\n"
result = strip_quoted_wrapping(input_text, "filename.ext")
self.assertEqual(result, expected_output)
def test_strip_quoted_wrapping_no_filename(self):
input_text = "```\nWe just want this content\nNot the triple quotes\n```"
expected_output = "We just want this content\nNot the triple quotes\n"
result = strip_quoted_wrapping(input_text)
self.assertEqual(result, expected_output)
def test_strip_quoted_wrapping_no_wrapping(self):
input_text = "We just want this content\nNot the triple quotes\n"
expected_output = "We just want this content\nNot the triple quotes\n"
result = strip_quoted_wrapping(input_text)
self.assertEqual(result, expected_output)
if __name__ == "__main__":
unittest.main()

109
aider/utils.py Normal file
View file

@ -0,0 +1,109 @@
import math
from difflib import SequenceMatcher
from pathlib import Path
# from dump import dump
def replace_most_similar_chunk(whole, part, replace):
similarity_thresh = 0.8
max_similarity = 0
most_similar_chunk_start = -1
most_similar_chunk_end = -1
whole_lines = whole.splitlines()
part_lines = part.splitlines()
scale = 0.1
min_len = math.floor(len(part_lines) * (1 - scale))
max_len = math.ceil(len(part_lines) * (1 + scale))
for length in range(min_len, max_len):
for i in range(len(whole_lines) - length + 1):
chunk = whole_lines[i : i + length + 1]
chunk = "\n".join(chunk)
similarity = SequenceMatcher(None, chunk, part).ratio()
if similarity > max_similarity and similarity:
max_similarity = similarity
most_similar_chunk_start = i
most_similar_chunk_end = i + length + 1
if max_similarity < similarity_thresh:
return
replace_lines = replace.splitlines()
modified_whole = (
whole_lines[:most_similar_chunk_start]
+ replace_lines
+ whole_lines[most_similar_chunk_end:]
)
modified_whole = "\n".join(modified_whole) + "\n"
return modified_whole
def quoted_file(fname):
prompt = "\n"
prompt += fname
prompt += "\n```\n"
prompt += Path(fname).read_text()
prompt += "\n```\n"
return prompt
def strip_quoted_wrapping(res, fname=None):
"""
Given an input string which may have extra "wrapping" around it, remove the wrapping.
For example:
filename.ext
```
We just want this content
Not the filename and triple quotes
```
"""
if not res:
return res
res = res.splitlines()
if fname and res[0].strip().endswith(Path(fname).name):
res = res[1:]
if res[0].startswith("```") and res[-1].startswith("```"):
res = res[1:-1]
res = "\n".join(res)
if res and res[-1] != "\n":
res += "\n"
return res
def do_replace(fname, before_text, after_text):
before_text = strip_quoted_wrapping(before_text, fname)
after_text = strip_quoted_wrapping(after_text, fname)
fname = Path(fname)
# does it want to make a new file?
if not fname.exists() and not before_text.strip():
fname.touch()
content = fname.read_text()
if not before_text.strip():
if content:
new_content = content + after_text
else:
# first populating an empty file
new_content = after_text
else:
new_content = replace_most_similar_chunk(content, before_text, after_text)
if not new_content:
return
fname.write_text(new_content)
return True