dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-06-05 04:05:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

refactor into Model.token_count()

Browse source
This commit is contained in:
Paul Gauthier 2023-11-14 10:11:13 -08:00
parent 90f57664a6
commit c24a4a4392
3 changed files with 24 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

13
aider/models/model.py
View file

@ -1,3 +1,5 @@
import json
import openai
@ -37,3 +39,14 @@ class Model:
@staticmethod
def commit_message_models():
return [Model.create("gpt-3.5-turbo"), Model.create("gpt-3.5-turbo-16k")]
def token_count(self, messages):
if not self.tokenizer:
return
if type(messages) is str:
msgs = messages
else:
msgs = json.dumps(messages)
return len(self.tokenizer.encode(msgs))