From c5ed12e0c32866be324b6827a13f863c8f807ada Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Fri, 2 Jun 2023 16:36:03 -0700 Subject: [PATCH] aider: Replaced shelve with diskcache for persistent dictionary-like object. # Context: USER: is there another persistent dictionary besides shelve? ASSISTANT: Yes, there are several alternatives to `shelve` for creating persistent dictionaries in Python. Some popular options include: 1. `pickle`: A built-in Python module for object serialization and deserialization. You can use it to store dictionaries in binary format and load them back into memory when needed. 2. `json`: Another built-in Python module for working with JSON data. You can use it to store dictionaries in a human-readable format and load them back into memory when needed. 3. `sqlite3`: A built-in Python module for working with SQLite databases. You can use it to create a simple key-value store with SQL queries. 4. `TinyDB`: A lightweight, document-oriented database for Python ^C KeyboardInterrupt USER: i want something which presents as a dict though ASSISTANT: In that case, you can use the `sqlite3` module to create a custom class that behaves like a dictionary. Here's an example of how you can create a persistent dictionary using SQLite: ```python import sqlite3 class ^C KeyboardInterrupt USER: i don't want to build something. i want an existing package. ASSISTANT: I apologize for the confusion. In that case, you can use the `diskcache` package, which provides a dictionary-like interface for disk-based storage. It is easy to use and has good performance. To install `diskcache`, you can use pip: ``` pip install diskcache ``` Here's an example of how to use `diskcache`: ```python from diskcache import Cache cache = Cache('path/to/cache/directory') # Set a key-value pair cache['key'] = 'value' # Get the value for a key value = cache['key'] # Check if a key is in the cache if 'key' in cache: print('Key exists') # Delete a key-value pair del cache['key'] # Close the cache when done cache.close() ``` This will create a persistent dictionary-like object using the `diskcache` package. The data will be stored on disk in the specified directory, and you can use the standard dictionary methods to interact with it. USER: great. use that. ASSISTANT: To use `diskcache` in the `aider/repomap.py` file, you'll need to replace the `shelve` usage with `diskcache`. Here are the changes you need to make: 1. Install `diskcache` using pip: ``` pip install diskcache ``` 2. Modify the `aider/repomap.py` file: ```python aider/repomap.py <<<<<<< ORIGINAL import colorsys import json import os import random import subprocess import sys import tempfile from collections import Counter, defaultdict # import shelve ======= import colorsys import json import os import random import subprocess import sys import tempfile from collections import Counter, defaultdict from diskcache import Cache >>>>>>> UPDATED ``` ```python aider/repomap.py <<<<<<< ORIGINAL IDENT_CACHE_FILE = ".aider.ident.cache" TAGS_CACHE_FILE = ".aider.tags.cache" ======= IDENT_CACHE_DIR = ".aider.ident.cache" TAGS_CACHE_DIR = ".aider.tags.cache" >>>>>>> UPDATED ``` ```python aider/repomap.py <<<<<<< ORIGINAL def load_tags_cache(self): self.TAGS_CACHE = dict() # shelve.open(self.TAGS_CACHE_FILE) def save_tags_cache(self): # self.TAGS_CACHE.sync() pass def load_ident_cache(self): self.IDENT_CACHE = dict() # shelve.open(self.IDENT_CACHE_FILE) def save_ident_cache(self): # self.IDENT_CACHE.sync() pass ======= def load_tags_cache(self): self.TAGS_CACHE = Cache(self.TAGS_CACHE_DIR) def save_tags_cache(self): self.TAGS_CACHE.close() def load_ident_cache(self): self.IDENT_CACHE = Cache(self.IDENT_CACHE_DIR) def save_ident_cache(self): self.IDENT_CACHE.close() >>>>>>> UPDATED ``` These changes will replace the usage of `shelve` with `diskcache` for the `IDENT_CACHE` and `TAGS_CACHE` dictionaries. Make sure to install the `diskcache` package before running the updated code. --- aider/repomap.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/aider/repomap.py b/aider/repomap.py index 00d3f5847..edd4d4482 100644 --- a/aider/repomap.py +++ b/aider/repomap.py @@ -6,8 +6,7 @@ import subprocess import sys import tempfile from collections import Counter, defaultdict - -# import shelve +from diskcache import Cache import networkx as nx import tiktoken from pygments.lexers import guess_lexer_for_filename @@ -55,8 +54,8 @@ def fname_to_components(fname, with_colon): class RepoMap: ctags_cmd = ["ctags", "--fields=+S", "--extras=-F", "--output-format=json"] - IDENT_CACHE_FILE = ".aider.ident.cache" - TAGS_CACHE_FILE = ".aider.tags.cache" + IDENT_CACHE_DIR = ".aider.ident.cache" + TAGS_CACHE_DIR = ".aider.tags.cache" # 1/4 of gpt-4's context window max_map_tokens = 512 @@ -167,18 +166,16 @@ class RepoMap: return True def load_tags_cache(self): - self.TAGS_CACHE = dict() # shelve.open(self.TAGS_CACHE_FILE) + self.TAGS_CACHE = Cache(self.TAGS_CACHE_DIR) def save_tags_cache(self): - # self.TAGS_CACHE.sync() - pass + self.TAGS_CACHE.close() def load_ident_cache(self): - self.IDENT_CACHE = dict() # shelve.open(self.IDENT_CACHE_FILE) + self.IDENT_CACHE = Cache(self.IDENT_CACHE_DIR) def save_ident_cache(self): - # self.IDENT_CACHE.sync() - pass + self.IDENT_CACHE.close() def get_name_identifiers(self, fname, uniq=True): file_mtime = os.path.getmtime(fname)