dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-20 12:24:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: Rename benchmark dir, improve rsync, fix problem stats

Browse source
This commit is contained in:
Paul Gauthier 2024-12-21 11:09:52 -08:00 • committed by Paul Gauthier (aider)
parent a75507980a
commit 4efdc8b4f7
4 changed files with 18 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

4
benchmark/clone-exercism.sh
View file

@ -1,10 +1,10 @@
#!/bin/bash #!/bin/bash
# Create directories if they don't exist # Create directories if they don't exist
mkdir -p tmp.benchmark/exercism mkdir -p tmp.benchmarks/exercism
# Change to the exercism directory # Change to the exercism directory
cd tmp.benchmark/exercism cd tmp.benchmarks/exercism
# List of languages to clone # List of languages to clone
languages=("cpp" "go" "java" "javascript" "python" "rust") languages=("cpp" "go" "java" "javascript" "python" "rust")

10
benchmark/install-docker-ubuntu.sh
View file

@ -49,11 +49,6 @@ fi
# Print success message and instructions # Print success message and instructions
echo "Docker installation completed successfully!" echo "Docker installation completed successfully!"
echo -e "\n*** IMPORTANT: Group changes require a complete session refresh."
echo "Please run these commands in sequence:"
echo "1. sudo systemctl restart docker"
echo "2. exit"
echo -e "Then log back in and docker commands should work without sudo\n"
# Start Docker service # Start Docker service
echo "Starting Docker service..." echo "Starting Docker service..."
@ -66,8 +61,3 @@ docker --version
echo "Docker Compose version:" echo "Docker Compose version:"
docker compose version docker compose version
echo "Docker service status:"
sudo systemctl status docker --no-pager
EOF

14
benchmark/problem_stats.py
View file

@ -23,9 +23,12 @@ def get_dirs_from_leaderboard():
def load_results(dirname): def load_results(dirname):
"""Load all result files from a benchmark directory""" """Load all result files from a benchmark directory"""
dirname = Path(dirname) dirname = Path(dirname)
benchmark_dir = Path("tmp.benchmarks") / dirname
benchmark_dir = dirname
if not benchmark_dir.exists(): if not benchmark_dir.exists():
return None benchmark_dir = Path("tmp.benchmarks") / dirname
if not benchmark_dir.exists():
return None
all_results = [] all_results = []
parse_errors = [] # Track which exercises had parse errors for this model parse_errors = [] # Track which exercises had parse errors for this model
@ -70,8 +73,11 @@ def analyze_exercise_solutions(dirs=None, topn=None, copy_hard_set=False):
valid_entries = [] valid_entries = []
parse_errors_by_model = {} # Track which exercises had parse errors for each model parse_errors_by_model = {} # Track which exercises had parse errors for each model
dump(dir_entries)
for dirname, model in dir_entries: for dirname, model in dir_entries:
results_data = load_results(dirname) results_data = load_results(dirname)
if results_data: if results_data:
results, model_parse_errors = results_data results, model_parse_errors = results_data
parse_errors_by_model[model] = set(model_parse_errors) parse_errors_by_model[model] = set(model_parse_errors)
@ -299,7 +305,7 @@ def analyze_exercise_solutions(dirs=None, topn=None, copy_hard_set=False):
if copy_hard_set: if copy_hard_set:
# Create hard set directory # Create hard set directory
src_dir = Path("tmp.benchmarks/exercism") src_dir = Path("tmp.benchmarks/exercism")
dst_dir = Path("tmp.benchmarks/exercism-hard-set") dst_dir = Path("tmp.benchmarks/exercism-polyglot")
if dst_dir.exists(): if dst_dir.exists():
print(f"\nError: Destination directory {dst_dir} already exists") print(f"\nError: Destination directory {dst_dir} already exists")
@ -340,7 +346,7 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
"--copy-hard-set", "--copy-hard-set",
action="store_true", action="store_true",
help="Copy hard set problems to tmp.benchmarks/exercism-hard-set", help="Copy hard set problems to tmp.benchmarks/exercism-polygot",
) )
args = parser.parse_args() args = parser.parse_args()

7
benchmark/rsync.sh
View file

@ -1,5 +1,7 @@
#!/bin/bash #!/bin/bash
set -e
if [ $# -ne 1 ]; then if [ $# -ne 1 ]; then
echo "Usage: $0 user@host" echo "Usage: $0 user@host"
exit 1 exit 1
@ -19,10 +21,13 @@ ssh "$DEST" "mkdir -p ~/aider"
# Sync the repository # Sync the repository
rsync -avz --delete \ rsync -avz --delete \
--exclude='.git/' \
--exclude-from="$EXCLUDE_FILE" \ --exclude-from="$EXCLUDE_FILE" \
"$REPO_ROOT/" \ "$REPO_ROOT/" \
"$DEST:~/aider/" "$DEST:~/aider/"
rsync -a .env .bash_history .gitignore "$DEST:~/aider/."
rsync -a ~/dotfiles/screenrc "$DEST:.screenrc"
# Clean up # Clean up
rm "$EXCLUDE_FILE" rm "$EXCLUDE_FILE"