From bafa8ab71c64839f120988147b29e309869f2878 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Sat, 13 Jul 2024 16:36:41 +0100 Subject: [PATCH] refactor reqs into requirements/ --- requirements.txt | 60 ++++++------ .../requirements-dev.in | 2 +- .../requirements-dev.txt | 68 ++++++------- .../requirements-hf-embed.in | 2 +- .../requirements-hf-embed.txt | 96 +++++++++---------- .../requirements.in | 0 scripts/pip-compile.sh | 15 ++- setup.py | 16 +++- 8 files changed, 137 insertions(+), 122 deletions(-) rename requirements-dev.in => requirements/requirements-dev.in (87%) rename requirements-dev.txt => requirements/requirements-dev.txt (65%) rename requirements-hf-embed.in => requirements/requirements-hf-embed.in (85%) rename requirements-hf-embed.txt => requirements/requirements-hf-embed.txt (66%) rename requirements.in => requirements/requirements.in (100%) diff --git a/requirements.txt b/requirements.txt index b6ec9f810..8176e9cf2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements.in +# pip-compile --output-file=requirements.txt requirements/requirements.in # aiohttp==3.9.5 # via litellm @@ -22,9 +22,9 @@ attrs==23.2.0 # jsonschema # referencing backoff==2.2.1 - # via -r requirements.in + # via -r requirements/requirements.in beautifulsoup4==4.12.3 - # via -r requirements.in + # via -r requirements/requirements.in blinker==1.8.2 # via streamlit cachetools==5.3.3 @@ -47,17 +47,17 @@ click==8.1.7 # litellm # streamlit configargparse==1.7 - # via -r requirements.in + # via -r requirements/requirements.in diff-match-patch==20230430 - # via -r requirements.in + # via -r requirements/requirements.in diskcache==5.6.3 - # via -r requirements.in + # via -r requirements/requirements.in distro==1.9.0 # via openai filelock==3.15.4 # via huggingface-hub flake8==7.1.0 - # via -r requirements.in + # via -r requirements/requirements.in frozenlist==1.4.1 # via # aiohttp @@ -68,7 +68,7 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.43 # via - # -r requirements.in + # -r requirements/requirements.in # streamlit google-ai-generativelanguage==0.6.6 # via google-generativeai @@ -89,7 +89,7 @@ google-auth==2.31.0 google-auth-httplib2==0.2.0 # via google-api-python-client google-generativeai==0.7.1 - # via -r requirements.in + # via -r requirements/requirements.in googleapis-common-protos==1.63.2 # via # google-api-core @@ -97,7 +97,7 @@ googleapis-common-protos==1.63.2 greenlet==3.0.3 # via playwright grep-ast==0.3.2 - # via -r requirements.in + # via -r requirements/requirements.in grpcio==1.64.1 # via # google-api-core @@ -124,10 +124,10 @@ idna==3.7 # yarl importlib-metadata==7.2.1 # via - # -r requirements.in + # -r requirements/requirements.in # litellm importlib-resources==6.4.0 - # via -r requirements.in + # via -r requirements/requirements.in jinja2==3.1.4 # via # altair @@ -135,13 +135,13 @@ jinja2==3.1.4 # pydeck jsonschema==4.22.0 # via - # -r requirements.in + # -r requirements/requirements.in # altair # litellm jsonschema-specifications==2023.12.1 # via jsonschema litellm==1.41.6 - # via -r requirements.in + # via -r requirements/requirements.in markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 @@ -155,10 +155,10 @@ multidict==6.0.5 # aiohttp # yarl networkx==3.2.1 - # via -r requirements.in + # via -r requirements/requirements.in numpy==1.26.4 # via - # -r requirements.in + # -r requirements/requirements.in # altair # pandas # pyarrow @@ -169,7 +169,7 @@ openai==1.35.10 # via litellm packaging==24.1 # via - # -r requirements.in + # -r requirements/requirements.in # altair # huggingface-hub # streamlit @@ -179,16 +179,16 @@ pandas==2.2.2 # streamlit pathspec==0.12.1 # via - # -r requirements.in + # -r requirements/requirements.in # grep-ast pillow==10.4.0 # via - # -r requirements.in + # -r requirements/requirements.in # streamlit playwright==1.45.0 - # via -r requirements.in + # via -r requirements/requirements.in prompt-toolkit==3.0.47 - # via -r requirements.in + # via -r requirements/requirements.in proto-plus==1.24.0 # via # google-ai-generativelanguage @@ -230,7 +230,7 @@ pyflakes==3.2.0 pygments==2.18.0 # via rich pypandoc==1.13 - # via -r requirements.in + # via -r requirements/requirements.in pyparsing==3.1.2 # via httplib2 python-dateutil==2.9.0.post0 @@ -241,7 +241,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via - # -r requirements.in + # -r requirements/requirements.in # huggingface-hub referencing==0.35.1 # via @@ -258,7 +258,7 @@ requests==2.32.3 # tiktoken rich==13.7.1 # via - # -r requirements.in + # -r requirements/requirements.in # streamlit rpds-py==0.18.1 # via @@ -267,7 +267,7 @@ rpds-py==0.18.1 rsa==4.9 # via google-auth scipy==1.13.1 - # via -r requirements.in + # via -r requirements/requirements.in six==1.16.0 # via python-dateutil smmap==5.0.1 @@ -278,13 +278,13 @@ sniffio==1.3.1 # httpx # openai sounddevice==0.4.7 - # via -r requirements.in + # via -r requirements/requirements.in soundfile==0.12.1 - # via -r requirements.in + # via -r requirements/requirements.in soupsieve==2.5 # via beautifulsoup4 streamlit==1.36.0 - # via -r requirements.in + # via -r requirements/requirements.in tenacity==8.4.2 # via streamlit tiktoken==0.7.0 @@ -304,7 +304,7 @@ tqdm==4.66.4 # openai tree-sitter==0.21.3 # via - # -r requirements.in + # -r requirements/requirements.in # tree-sitter-languages tree-sitter-languages==1.10.2 # via grep-ast @@ -324,7 +324,7 @@ uritemplate==4.1.1 urllib3==2.2.2 # via requests watchdog==4.0.1 - # via -r requirements.in + # via -r requirements/requirements.in wcwidth==0.2.13 # via prompt-toolkit yarl==1.9.4 diff --git a/requirements-dev.in b/requirements/requirements-dev.in similarity index 87% rename from requirements-dev.in rename to requirements/requirements-dev.in index f1d040cd2..f2c3badfe 100644 --- a/requirements-dev.in +++ b/requirements/requirements-dev.in @@ -1,4 +1,4 @@ --c requirements.txt +-c ../requirements.txt # # pip-compile --output-file=requirements-dev.txt requirements-dev.in --upgrade # diff --git a/requirements-dev.txt b/requirements/requirements-dev.txt similarity index 65% rename from requirements-dev.txt rename to requirements/requirements-dev.txt index 92f5833d2..4c74677fb 100644 --- a/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --output-file=requirements-dev.txt requirements-dev.in +# pip-compile --output-file=requirements/requirements-dev.txt requirements/requirements-dev.in # alabaster==0.7.16 # via sphinx @@ -12,21 +12,21 @@ build==1.2.1 # via pip-tools certifi==2024.7.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # requests cfgv==3.4.0 # via pre-commit charset-normalizer==3.3.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # requests click==8.1.7 # via - # -c requirements.txt + # -c requirements/../requirements.txt # pip-tools # typer cogapp==3.4.1 - # via -r requirements-dev.in + # via -r requirements/requirements-dev.in contourpy==1.2.1 # via matplotlib cycler==0.12.1 @@ -43,7 +43,7 @@ docutils==0.20.1 # sphinx-rtd-theme filelock==3.15.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # virtualenv fonttools==4.53.1 # via matplotlib @@ -51,35 +51,35 @@ identify==2.5.36 # via pre-commit idna==3.7 # via - # -c requirements.txt + # -c requirements/../requirements.txt # requests imagesize==1.4.1 # via sphinx imgcat==0.5.0 - # via -r requirements-dev.in + # via -r requirements/requirements-dev.in iniconfig==2.0.0 # via pytest jinja2==3.1.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # sphinx kiwisolver==1.4.5 # via matplotlib lox==0.12.0 - # via -r requirements-dev.in + # via -r requirements/requirements-dev.in markdown-it-py==3.0.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # rich markupsafe==2.1.5 # via - # -c requirements.txt + # -c requirements/../requirements.txt # jinja2 matplotlib==3.9.1 - # via -r requirements-dev.in + # via -r requirements/requirements-dev.in mdurl==0.1.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # markdown-it-py multiprocess==0.70.16 # via pathos @@ -87,29 +87,29 @@ nodeenv==1.9.1 # via pre-commit numpy==1.26.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # contourpy # matplotlib # pandas packaging==24.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # build # matplotlib # pytest # sphinx pandas==2.2.2 # via - # -c requirements.txt - # -r requirements-dev.in + # -c requirements/../requirements.txt + # -r requirements/requirements-dev.in pathos==0.3.2 # via lox pillow==10.4.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # matplotlib pip-tools==7.4.1 - # via -r requirements-dev.in + # via -r requirements/requirements-dev.in platformdirs==4.2.2 # via virtualenv pluggy==1.5.0 @@ -119,48 +119,48 @@ pox==0.3.4 ppft==1.7.6.8 # via pathos pre-commit==3.7.1 - # via -r requirements-dev.in + # via -r requirements/requirements-dev.in pygments==2.18.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # rich # sphinx pyparsing==3.1.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # matplotlib pyproject-hooks==1.1.0 # via # build # pip-tools pytest==8.2.2 - # via -r requirements-dev.in + # via -r requirements/requirements-dev.in python-dateutil==2.9.0.post0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # matplotlib # pandas pytz==2024.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # pandas pyyaml==6.0.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # pre-commit requests==2.32.3 # via - # -c requirements.txt + # -c requirements/../requirements.txt # sphinx rich==13.7.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # typer shellingham==1.5.4 # via typer six==1.16.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # python-dateutil snowballstemmer==2.2.0 # via sphinx @@ -185,18 +185,18 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx typer==0.12.3 - # via -r requirements-dev.in + # via -r requirements/requirements-dev.in typing-extensions==4.12.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # typer tzdata==2024.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # pandas urllib3==2.2.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # requests virtualenv==20.26.3 # via pre-commit diff --git a/requirements-hf-embed.in b/requirements/requirements-hf-embed.in similarity index 85% rename from requirements-hf-embed.in rename to requirements/requirements-hf-embed.in index 38df32543..8b10df15f 100644 --- a/requirements-hf-embed.in +++ b/requirements/requirements-hf-embed.in @@ -1,4 +1,4 @@ --c requirements.txt +-c ../requirements.txt # # pip-compile --output-file=requirements-hf.txt requirements-hf.in --upgrade # diff --git a/requirements-hf-embed.txt b/requirements/requirements-hf-embed.txt similarity index 66% rename from requirements-hf-embed.txt rename to requirements/requirements-hf-embed.txt index 257941b83..6911de143 100644 --- a/requirements-hf-embed.txt +++ b/requirements/requirements-hf-embed.txt @@ -2,43 +2,43 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --output-file=requirements-hf-embed.txt requirements-hf-embed.in +# pip-compile --output-file=requirements/requirements-hf-embed.txt requirements/requirements-hf-embed.in # aiohttp==3.9.5 # via - # -c requirements.txt + # -c requirements/../requirements.txt # huggingface-hub # llama-index-core aiosignal==1.3.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # aiohttp annotated-types==0.7.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # pydantic anyio==4.4.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # httpx # openai attrs==23.2.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # aiohttp certifi==2024.7.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # httpcore # httpx # requests charset-normalizer==3.3.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # requests click==8.1.7 # via - # -c requirements.txt + # -c requirements/../requirements.txt # nltk dataclasses-json==0.6.7 # via llama-index-core @@ -48,60 +48,60 @@ dirtyjson==1.0.8 # via llama-index-core distro==1.9.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # openai filelock==3.15.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # huggingface-hub # torch # transformers frozenlist==1.4.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # aiohttp # aiosignal fsspec==2024.6.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # huggingface-hub # llama-index-core # torch greenlet==3.0.3 # via - # -c requirements.txt + # -c requirements/../requirements.txt # sqlalchemy h11==0.14.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # httpcore httpcore==1.0.5 # via - # -c requirements.txt + # -c requirements/../requirements.txt # httpx httpx==0.27.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-cloud # llama-index-core # openai huggingface-hub[inference]==0.23.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-index-embeddings-huggingface # sentence-transformers # tokenizers # transformers idna==3.7 # via - # -c requirements.txt + # -c requirements/../requirements.txt # anyio # httpx # requests # yarl jinja2==3.1.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # torch joblib==1.4.2 # via @@ -111,13 +111,13 @@ llama-cloud==0.0.6 # via llama-index-core llama-index-core==0.10.52.post2 # via - # -r requirements-hf-embed.in + # -r requirements/requirements-hf-embed.in # llama-index-embeddings-huggingface llama-index-embeddings-huggingface==0.2.2 - # via -r requirements-hf-embed.in + # via -r requirements/requirements-hf-embed.in markupsafe==2.1.5 # via - # -c requirements.txt + # -c requirements/../requirements.txt # jinja2 marshmallow==3.21.3 # via dataclasses-json @@ -127,7 +127,7 @@ mpmath==1.3.0 # via sympy multidict==6.0.5 # via - # -c requirements.txt + # -c requirements/../requirements.txt # aiohttp # yarl mypy-extensions==1.0.0 @@ -136,14 +136,14 @@ nest-asyncio==1.6.0 # via llama-index-core networkx==3.2.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-index-core # torch nltk==3.8.1 # via llama-index-core numpy==1.26.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-index-core # pandas # scikit-learn @@ -152,55 +152,55 @@ numpy==1.26.4 # transformers openai==1.35.10 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-index-core packaging==24.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # huggingface-hub # marshmallow # transformers pandas==2.2.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-index-core pillow==10.4.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-index-core # sentence-transformers pydantic==2.8.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-cloud # openai pydantic-core==2.20.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # pydantic python-dateutil==2.9.0.post0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # pandas pytz==2024.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # pandas pyyaml==6.0.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # huggingface-hub # llama-index-core # transformers regex==2024.5.15 # via - # -c requirements.txt + # -c requirements/../requirements.txt # nltk # tiktoken # transformers requests==2.32.3 # via - # -c requirements.txt + # -c requirements/../requirements.txt # huggingface-hub # llama-index-core # tiktoken @@ -211,18 +211,18 @@ scikit-learn==1.5.1 # via sentence-transformers scipy==1.13.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # scikit-learn # sentence-transformers sentence-transformers==3.0.1 # via llama-index-embeddings-huggingface six==1.16.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # python-dateutil sniffio==1.3.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # anyio # httpx # openai @@ -234,23 +234,23 @@ sympy==1.13.0 # via torch tenacity==8.4.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-index-core threadpoolctl==3.5.0 # via scikit-learn tiktoken==0.7.0 # via - # -c requirements.txt + # -c requirements/../requirements.txt # llama-index-core tokenizers==0.19.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # transformers torch==2.2.2 # via sentence-transformers tqdm==4.66.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # huggingface-hub # llama-index-core # nltk @@ -261,7 +261,7 @@ transformers==4.42.3 # via sentence-transformers typing-extensions==4.12.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # huggingface-hub # llama-index-core # openai @@ -276,11 +276,11 @@ typing-inspect==0.9.0 # llama-index-core tzdata==2024.1 # via - # -c requirements.txt + # -c requirements/../requirements.txt # pandas urllib3==2.2.2 # via - # -c requirements.txt + # -c requirements/../requirements.txt # requests wrapt==1.16.0 # via @@ -288,5 +288,5 @@ wrapt==1.16.0 # llama-index-core yarl==1.9.4 # via - # -c requirements.txt + # -c requirements/../requirements.txt # aiohttp diff --git a/requirements.in b/requirements/requirements.in similarity index 100% rename from requirements.in rename to requirements/requirements.in diff --git a/scripts/pip-compile.sh b/scripts/pip-compile.sh index 956bac933..3ee55def4 100755 --- a/scripts/pip-compile.sh +++ b/scripts/pip-compile.sh @@ -3,7 +3,16 @@ # exit when any command fails set -e -pip-compile requirements.in $1 -pip-compile --output-file=requirements-dev.txt requirements-dev.in $1 -pip-compile --output-file=requirements-hf-embed.txt requirements-hf-embed.in $1 +pip-compile \ + requirements/requirements.in \ + --output-file=requirements.txt \ + $1 +for SUFFIX in dev hf-embed ; do + echo suffix: ${SUFFIX} + pip-compile \ + requirements/requirements-${SUFFIX}.in \ + --output-file=requirements/requirements-${SUFFIX}.txt \ + $1 +done + diff --git a/setup.py b/setup.py index 9afc96374..67e64263d 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import re +from pathlib import Path from setuptools import find_packages, setup @@ -7,15 +8,20 @@ from aider.help_pats import exclude_website_pats def get_requirements(suffix=""): - fname = "requirements" + suffix + ".txt" - with open(fname) as f: - requirements = f.read().splitlines() + if suffix: + fname = "requirements-" + suffix + ".txt" + fname = Path("requirements") / fname + else: + fname = Path("requirements.txt") + + requirements = fname.read_text().splitlines() + return requirements requirements = get_requirements() -dev_requirements = get_requirements("-dev") -hf_requirements = get_requirements("-hf-embed") +dev_requirements = get_requirements("dev") +hf_requirements = get_requirements("hf-embed") # README with open("README.md", "r", encoding="utf-8") as f: