diff --git a/aider/repomap.py b/aider/repomap.py index 631024a19..e475b8698 100644 --- a/aider/repomap.py +++ b/aider/repomap.py @@ -7,8 +7,11 @@ from collections import Counter, defaultdict import networkx as nx import tiktoken +from pygments.lexers import guess_lexer_for_filename +from pygments.token import Token +from pygments.util import ClassNotFound -from aider import prompts, utils +from aider import prompts # from aider.dump import dump @@ -50,6 +53,7 @@ def fname_to_components(fname, with_colon): class RepoMap: ctags_cmd = ["ctags", "--fields=+S", "--extras=-F", "--output-format=json"] TAGS_CACHE = None + IDENT_CACHE = None def __init__(self, use_ctags=None, root=None, main_model="gpt-4"): if not root: @@ -57,6 +61,7 @@ class RepoMap: self.root = root self.TAGS_CACHE = dict() + self.IDENT_CACHE = dict() if use_ctags is None: self.use_ctags = self.check_for_ctags() @@ -189,6 +194,32 @@ class RepoMap: return False return True + def get_name_identifiers(self, fname, uniq=True): + cached = self.IDENT_CACHE.get(fname) + if cached is not None: + if uniq: + cached = set(cached) + return cached + + with open(fname, "r") as f: + content = f.read() + try: + lexer = guess_lexer_for_filename(fname, content) + except ClassNotFound: + self.IDENT_CACHE[fname] = list() + return list() + + # lexer.get_tokens_unprocessed() returns (char position in file, token type, token string) + tokens = list(lexer.get_tokens_unprocessed(content)) + res = [token[2] for token in tokens if token[1] in Token.Name] + + self.IDENT_CACHE[fname] = res + + if uniq: + res = set(res) + + return res + def find_py_files(directory): if not os.path.isdir(directory): @@ -239,7 +270,7 @@ def call_map(): defines[ident].add(show_fname) # dump("def", fname, ident) - idents = utils.get_name_identifiers(fname, uniq=False) + idents = rm.get_name_identifiers(fname, uniq=False) for ident in idents: # dump("ref", fname, ident) references[ident].append(show_fname) diff --git a/aider/utils.py b/aider/utils.py index a1ee8123a..d758340dd 100644 --- a/aider/utils.py +++ b/aider/utils.py @@ -3,10 +3,6 @@ import re from difflib import SequenceMatcher from pathlib import Path -from pygments.lexers import guess_lexer_for_filename -from pygments.token import Token -from pygments.util import ClassNotFound - # from aider.dump import dump @@ -284,21 +280,6 @@ def find_original_update_blocks(content): raise ValueError(f"{processed}\n^^^ Error parsing ORIGINAL/UPDATED block.") -def get_name_identifiers(fname, uniq=True): - with open(fname, "r") as f: - content = f.read() - try: - lexer = guess_lexer_for_filename(fname, content) - except ClassNotFound: - return list() - # lexer.get_tokens_unprocessed() returns (char position in file, token type, token string) - tokens = list(lexer.get_tokens_unprocessed(content)) - res = [token[2] for token in tokens if token[1] in Token.Name] - if uniq: - res = set(res) - return res - - if __name__ == "__main__": edit = """ Here's the change: