From 90f57664a65c91be1e53dbaeb066cc70e60eb5f8 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Tue, 14 Nov 2023 10:04:00 -0800 Subject: [PATCH] aider: Refactor `cmd_tokens` to use `main_model.token_count()` instead of calling tokenizer directly. # Aider chat conversation: USER: in `cmd_tokens` make use of main_model.token_count() rather than calling tokenizer directly ASSISTANT: To make this change, we need to modify `aider/commands.py`. Specifically, we will update the `cmd_tokens` method to use `self.coder.main_model.token_count()` instead of calling `self.tokenizer.encode()` directly. Here is the *SEARCH/REPLACE* block for the change: aider/commands.py python <<<<<<< SEARCH tokens = len(self.tokenizer.encode(json.dumps(msgs))) ======= tokens = self.coder.main_model.token_count(json.dumps(msgs)) >>>>>>> REPLACE aider/commands.py python <<<<<<< SEARCH tokens = len(self.tokenizer.encode(msgs)) ======= tokens = self.coder.main_model.token_count(msgs) >>>>>>> REPLACE aider/commands.py python <<<<<<< SEARCH tokens = len(self.tokenizer.encode(repo_content)) ======= tokens = self.coder.main_model.token_count(repo_content) >>>>>>> REPLACE aider/commands.py python <<<<<<< SEARCH tokens = len(self.tokenizer.encode(content)) ======= tokens = self.coder.main_model.token_count(content) >>>>>>> REPLACE These changes will ensure that the `cmd_tokens` method uses the `token_count` method from the `main_model` to calculate the number of tokens, which is a more abstracted and potentially model-specific way of counting tokens. --- aider/commands.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aider/commands.py b/aider/commands.py index e99840dc9..c4e9f4b15 100644 --- a/aider/commands.py +++ b/aider/commands.py @@ -109,7 +109,7 @@ class Commands: dict(role="system", content=self.coder.gpt_prompts.main_system), dict(role="system", content=self.coder.gpt_prompts.system_reminder), ] - tokens = len(self.tokenizer.encode(json.dumps(msgs))) + tokens = self.coder.main_model.token_count(json.dumps(msgs)) res.append((tokens, "system messages", "")) # chat history @@ -117,7 +117,7 @@ class Commands: if msgs: msgs = [dict(role="dummy", content=msg) for msg in msgs] msgs = json.dumps(msgs) - tokens = len(self.tokenizer.encode(msgs)) + tokens = self.coder.main_model.token_count(msgs) res.append((tokens, "chat history", "use /clear to clear")) # repo map @@ -125,7 +125,7 @@ class Commands: if self.coder.repo_map: repo_content = self.coder.repo_map.get_repo_map(self.coder.abs_fnames, other_files) if repo_content: - tokens = len(self.tokenizer.encode(repo_content)) + tokens = self.coder.main_model.token_count(repo_content) res.append((tokens, "repository map", "use --map-tokens to resize")) # files @@ -134,7 +134,7 @@ class Commands: content = self.io.read_text(fname) # approximate content = f"{relative_fname}\n```\n" + content + "```\n" - tokens = len(self.tokenizer.encode(content)) + tokens = self.coder.main_model.token_count(content) res.append((tokens, f"{relative_fname}", "use /drop to drop from chat")) self.io.tool_output("Approximate context window usage, in tokens:")