dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-06-01 02:05:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

fix: Ensure token count calculation handles small text samples

Browse source
This commit is contained in:
Paul Gauthier 2024-08-05 17:14:42 -03:00 • committed by Paul Gauthier (aider)
parent 75c3c40354
commit b6760e26c7
1 changed files with 5 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
aider/repomap.py
View file

@ -79,7 +79,11 @@ class RepoMap:
return len(text) / self.tokens_per_char return len(text) / self.tokens_per_char
sample_text = text.splitlines(keepends=True) sample_text = text.splitlines(keepends=True)
sample_text = "".join(random.sample(sample_text, 150)) samples = 150
if len(sample_text) < samples:
return self.main_model.token_count(text)
sample_text = "".join(random.sample(sample_text, samples))
tokens = self.main_model.token_count(sample_text) tokens = self.main_model.token_count(sample_text)
self.tokens_per_char = tokens / len(sample_text) self.tokens_per_char = tokens / len(sample_text)