dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-06-04 11:45:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

refactor tokenizer

Browse source
This commit is contained in:
Paul Gauthier 2024-04-19 12:08:35 -07:00
parent 6f1cebc4c2
commit 547ae142ba
3 changed files with 4 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

7
aider/history.py
View file

@ -1,5 +1,4 @@
import argparse
import json
from aider import models, prompts
from aider.dump import dump # noqa: F401
@ -8,7 +7,7 @@ from aider.sendchat import simple_send_with_retries
class ChatSummary:
def __init__(self, model=None, max_tokens=1024):
self.tokenizer = model.tokenizer
self.token_count = model.token_count
self.max_tokens = max_tokens
self.model = model
@ -20,7 +19,7 @@ class ChatSummary:
def tokenize(self, messages):
sized = []
for msg in messages:
tokens = len(self.tokenizer(json.dumps(msg)))
tokens = self.token_count(msg)
sized.append((tokens, msg))
return sized
@ -60,7 +59,7 @@ class ChatSummary:
summary = self.summarize_all(head)
tail_tokens = sum(tokens for tokens, msg in sized[split_index:])
summary_tokens = len(self.tokenizer(json.dumps(summary)))
summary_tokens = self.token_count(summary)
result = summary + tail
if summary_tokens + tail_tokens < self.max_tokens: