Merge pull request #3127 from FeepingCreature/fix/sort-file-tokens

refactor: Sort files by token count in cmd_tokens
This commit is contained in:
paul-gauthier 2025-02-18 11:50:36 -08:00 committed by GitHub
commit 86175a1827
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -404,6 +404,7 @@ class Commands:
fence = "`" * 3
file_res = []
# files
for fname in self.coder.abs_fnames:
relative_fname = self.coder.get_rel_fname(fname)
@ -414,7 +415,7 @@ class Commands:
# approximate
content = f"{relative_fname}\n{fence}\n" + content + "{fence}\n"
tokens = self.coder.main_model.token_count(content)
res.append((tokens, f"{relative_fname}", "/drop to remove"))
file_res.append((tokens, f"{relative_fname}", "/drop to remove"))
# read-only files
for fname in self.coder.abs_read_only_fnames:
@ -424,7 +425,10 @@ class Commands:
# approximate
content = f"{relative_fname}\n{fence}\n" + content + "{fence}\n"
tokens = self.coder.main_model.token_count(content)
res.append((tokens, f"{relative_fname} (read-only)", "/drop to remove"))
file_res.append((tokens, f"{relative_fname} (read-only)", "/drop to remove"))
file_res.sort()
res.extend(file_res)
self.io.tool_output(
f"Approximate context window usage for {self.coder.main_model.name}, in tokens:"