refactor reqs into requirements/

requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile --output-file=requirements.txt requirements/requirements.in
 #
 aiohttp==3.9.5
     # via litellm
@@ -22,9 +22,9 @@ attrs==23.2.0
     #   jsonschema
     #   referencing
 backoff==2.2.1
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 beautifulsoup4==4.12.3
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 blinker==1.8.2
     # via streamlit
 cachetools==5.3.3
@@ -47,17 +47,17 @@ click==8.1.7
     #   litellm
     #   streamlit
 configargparse==1.7
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 diff-match-patch==20230430
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 diskcache==5.6.3
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 distro==1.9.0
     # via openai
 filelock==3.15.4
     # via huggingface-hub
 flake8==7.1.0
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 frozenlist==1.4.1
     # via
     #   aiohttp
@@ -68,7 +68,7 @@ gitdb==4.0.11
     # via gitpython
 gitpython==3.1.43
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   streamlit
 google-ai-generativelanguage==0.6.6
     # via google-generativeai
@@ -89,7 +89,7 @@ google-auth==2.31.0
 google-auth-httplib2==0.2.0
     # via google-api-python-client
 google-generativeai==0.7.1
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 googleapis-common-protos==1.63.2
     # via
     #   google-api-core
@@ -97,7 +97,7 @@ googleapis-common-protos==1.63.2
 greenlet==3.0.3
     # via playwright
 grep-ast==0.3.2
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 grpcio==1.64.1
     # via
     #   google-api-core
@@ -124,10 +124,10 @@ idna==3.7
     #   yarl
 importlib-metadata==7.2.1
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   litellm
 importlib-resources==6.4.0
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 jinja2==3.1.4
     # via
     #   altair
@@ -135,13 +135,13 @@ jinja2==3.1.4
     #   pydeck
 jsonschema==4.22.0
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   altair
     #   litellm
 jsonschema-specifications==2023.12.1
     # via jsonschema
 litellm==1.41.6
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 markdown-it-py==3.0.0
     # via rich
 markupsafe==2.1.5
@@ -155,10 +155,10 @@ multidict==6.0.5
     #   aiohttp
     #   yarl
 networkx==3.2.1
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 numpy==1.26.4
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   altair
     #   pandas
     #   pyarrow
@@ -169,7 +169,7 @@ openai==1.35.10
     # via litellm
 packaging==24.1
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   altair
     #   huggingface-hub
     #   streamlit
@@ -179,16 +179,16 @@ pandas==2.2.2
     #   streamlit
 pathspec==0.12.1
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   grep-ast
 pillow==10.4.0
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   streamlit
 playwright==1.45.0
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 prompt-toolkit==3.0.47
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 proto-plus==1.24.0
     # via
     #   google-ai-generativelanguage
@@ -230,7 +230,7 @@ pyflakes==3.2.0
 pygments==2.18.0
     # via rich
 pypandoc==1.13
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 pyparsing==3.1.2
     # via httplib2
 python-dateutil==2.9.0.post0
@@ -241,7 +241,7 @@ pytz==2024.1
     # via pandas
 pyyaml==6.0.1
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   huggingface-hub
 referencing==0.35.1
     # via
@@ -258,7 +258,7 @@ requests==2.32.3
     #   tiktoken
 rich==13.7.1
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   streamlit
 rpds-py==0.18.1
     # via
@@ -267,7 +267,7 @@ rpds-py==0.18.1
 rsa==4.9
     # via google-auth
 scipy==1.13.1
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 six==1.16.0
     # via python-dateutil
 smmap==5.0.1
@@ -278,13 +278,13 @@ sniffio==1.3.1
     #   httpx
     #   openai
 sounddevice==0.4.7
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 soundfile==0.12.1
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 soupsieve==2.5
     # via beautifulsoup4
 streamlit==1.36.0
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 tenacity==8.4.2
     # via streamlit
 tiktoken==0.7.0
@@ -304,7 +304,7 @@ tqdm==4.66.4
     #   openai
 tree-sitter==0.21.3
     # via
-    #   -r requirements.in
+    #   -r requirements/requirements.in
     #   tree-sitter-languages
 tree-sitter-languages==1.10.2
     # via grep-ast
@@ -324,7 +324,7 @@ uritemplate==4.1.1
 urllib3==2.2.2
     # via requests
 watchdog==4.0.1
-    # via -r requirements.in
+    # via -r requirements/requirements.in
 wcwidth==0.2.13
     # via prompt-toolkit
 yarl==1.9.4

requirements/requirements-dev.in

@@ -1,4 +1,4 @@
--c requirements.txt
+-c ../requirements.txt
 #
 #    pip-compile --output-file=requirements-dev.txt requirements-dev.in --upgrade
 #

requirements/requirements-dev.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile --output-file=requirements-dev.txt requirements-dev.in
+#    pip-compile --output-file=requirements/requirements-dev.txt requirements/requirements-dev.in
 #
 alabaster==0.7.16
     # via sphinx
@@ -12,21 +12,21 @@ build==1.2.1
     # via pip-tools
 certifi==2024.7.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   requests
 cfgv==3.4.0
     # via pre-commit
 charset-normalizer==3.3.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   requests
 click==8.1.7
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   pip-tools
     #   typer
 cogapp==3.4.1
-    # via -r requirements-dev.in
+    # via -r requirements/requirements-dev.in
 contourpy==1.2.1
     # via matplotlib
 cycler==0.12.1
@@ -43,7 +43,7 @@ docutils==0.20.1
     #   sphinx-rtd-theme
 filelock==3.15.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   virtualenv
 fonttools==4.53.1
     # via matplotlib
@@ -51,35 +51,35 @@ identify==2.5.36
     # via pre-commit
 idna==3.7
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   requests
 imagesize==1.4.1
     # via sphinx
 imgcat==0.5.0
-    # via -r requirements-dev.in
+    # via -r requirements/requirements-dev.in
 iniconfig==2.0.0
     # via pytest
 jinja2==3.1.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   sphinx
 kiwisolver==1.4.5
     # via matplotlib
 lox==0.12.0
-    # via -r requirements-dev.in
+    # via -r requirements/requirements-dev.in
 markdown-it-py==3.0.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   rich
 markupsafe==2.1.5
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   jinja2
 matplotlib==3.9.1
-    # via -r requirements-dev.in
+    # via -r requirements/requirements-dev.in
 mdurl==0.1.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   markdown-it-py
 multiprocess==0.70.16
     # via pathos
@@ -87,29 +87,29 @@ nodeenv==1.9.1
     # via pre-commit
 numpy==1.26.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   contourpy
     #   matplotlib
     #   pandas
 packaging==24.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   build
     #   matplotlib
     #   pytest
     #   sphinx
 pandas==2.2.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
-    #   -r requirements-dev.in
+    #   -r requirements/requirements-dev.in
 pathos==0.3.2
     # via lox
 pillow==10.4.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   matplotlib
 pip-tools==7.4.1
-    # via -r requirements-dev.in
+    # via -r requirements/requirements-dev.in
 platformdirs==4.2.2
     # via virtualenv
 pluggy==1.5.0
@@ -119,48 +119,48 @@ pox==0.3.4
 ppft==1.7.6.8
     # via pathos
 pre-commit==3.7.1
-    # via -r requirements-dev.in
+    # via -r requirements/requirements-dev.in
 pygments==2.18.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   rich
     #   sphinx
 pyparsing==3.1.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   matplotlib
 pyproject-hooks==1.1.0
     # via
     #   build
     #   pip-tools
 pytest==8.2.2
-    # via -r requirements-dev.in
+    # via -r requirements/requirements-dev.in
 python-dateutil==2.9.0.post0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   matplotlib
     #   pandas
 pytz==2024.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   pandas
 pyyaml==6.0.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   pre-commit
 requests==2.32.3
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   sphinx
 rich==13.7.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   typer
 shellingham==1.5.4
     # via typer
 six==1.16.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   python-dateutil
 snowballstemmer==2.2.0
     # via sphinx
@@ -185,18 +185,18 @@ sphinxcontrib-qthelp==1.0.7
 sphinxcontrib-serializinghtml==1.1.10
     # via sphinx
 typer==0.12.3
-    # via -r requirements-dev.in
+    # via -r requirements/requirements-dev.in
 typing-extensions==4.12.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   typer
 tzdata==2024.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   pandas
 urllib3==2.2.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   requests
 virtualenv==20.26.3
     # via pre-commit

requirements/requirements-hf-embed.in

@@ -1,4 +1,4 @@
--c requirements.txt
+-c ../requirements.txt
 #
 #    pip-compile --output-file=requirements-hf.txt requirements-hf.in --upgrade
 #

requirements/requirements-hf-embed.txt

@@ -2,43 +2,43 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile --output-file=requirements-hf-embed.txt requirements-hf-embed.in
+#    pip-compile --output-file=requirements/requirements-hf-embed.txt requirements/requirements-hf-embed.in
 #
 aiohttp==3.9.5
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   huggingface-hub
     #   llama-index-core
 aiosignal==1.3.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   aiohttp
 annotated-types==0.7.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   pydantic
 anyio==4.4.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   httpx
     #   openai
 attrs==23.2.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   aiohttp
 certifi==2024.7.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   httpcore
     #   httpx
     #   requests
 charset-normalizer==3.3.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   requests
 click==8.1.7
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   nltk
 dataclasses-json==0.6.7
     # via llama-index-core
@@ -48,60 +48,60 @@ dirtyjson==1.0.8
     # via llama-index-core
 distro==1.9.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   openai
 filelock==3.15.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   huggingface-hub
     #   torch
     #   transformers
 frozenlist==1.4.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   aiohttp
     #   aiosignal
 fsspec==2024.6.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   huggingface-hub
     #   llama-index-core
     #   torch
 greenlet==3.0.3
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   sqlalchemy
 h11==0.14.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   httpcore
 httpcore==1.0.5
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   httpx
 httpx==0.27.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-cloud
     #   llama-index-core
     #   openai
 huggingface-hub[inference]==0.23.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-index-embeddings-huggingface
     #   sentence-transformers
     #   tokenizers
     #   transformers
 idna==3.7
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   anyio
     #   httpx
     #   requests
     #   yarl
 jinja2==3.1.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   torch
 joblib==1.4.2
     # via
@@ -111,13 +111,13 @@ llama-cloud==0.0.6
     # via llama-index-core
 llama-index-core==0.10.52.post2
     # via
-    #   -r requirements-hf-embed.in
+    #   -r requirements/requirements-hf-embed.in
     #   llama-index-embeddings-huggingface
 llama-index-embeddings-huggingface==0.2.2
-    # via -r requirements-hf-embed.in
+    # via -r requirements/requirements-hf-embed.in
 markupsafe==2.1.5
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   jinja2
 marshmallow==3.21.3
     # via dataclasses-json
@@ -127,7 +127,7 @@ mpmath==1.3.0
     # via sympy
 multidict==6.0.5
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   aiohttp
     #   yarl
 mypy-extensions==1.0.0
@@ -136,14 +136,14 @@ nest-asyncio==1.6.0
     # via llama-index-core
 networkx==3.2.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-index-core
     #   torch
 nltk==3.8.1
     # via llama-index-core
 numpy==1.26.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-index-core
     #   pandas
     #   scikit-learn
@@ -152,55 +152,55 @@ numpy==1.26.4
     #   transformers
 openai==1.35.10
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-index-core
 packaging==24.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   huggingface-hub
     #   marshmallow
     #   transformers
 pandas==2.2.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-index-core
 pillow==10.4.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-index-core
     #   sentence-transformers
 pydantic==2.8.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-cloud
     #   openai
 pydantic-core==2.20.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   pydantic
 python-dateutil==2.9.0.post0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   pandas
 pytz==2024.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   pandas
 pyyaml==6.0.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   huggingface-hub
     #   llama-index-core
     #   transformers
 regex==2024.5.15
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   nltk
     #   tiktoken
     #   transformers
 requests==2.32.3
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   huggingface-hub
     #   llama-index-core
     #   tiktoken
@@ -211,18 +211,18 @@ scikit-learn==1.5.1
     # via sentence-transformers
 scipy==1.13.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   scikit-learn
     #   sentence-transformers
 sentence-transformers==3.0.1
     # via llama-index-embeddings-huggingface
 six==1.16.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   python-dateutil
 sniffio==1.3.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   anyio
     #   httpx
     #   openai
@@ -234,23 +234,23 @@ sympy==1.13.0
     # via torch
 tenacity==8.4.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-index-core
 threadpoolctl==3.5.0
     # via scikit-learn
 tiktoken==0.7.0
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   llama-index-core
 tokenizers==0.19.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   transformers
 torch==2.2.2
     # via sentence-transformers
 tqdm==4.66.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   huggingface-hub
     #   llama-index-core
     #   nltk
@@ -261,7 +261,7 @@ transformers==4.42.3
     # via sentence-transformers
 typing-extensions==4.12.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   huggingface-hub
     #   llama-index-core
     #   openai
@@ -276,11 +276,11 @@ typing-inspect==0.9.0
     #   llama-index-core
 tzdata==2024.1
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   pandas
 urllib3==2.2.2
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   requests
 wrapt==1.16.0
     # via
@@ -288,5 +288,5 @@ wrapt==1.16.0
     #   llama-index-core
 yarl==1.9.4
     # via
-    #   -c requirements.txt
+    #   -c requirements/../requirements.txt
     #   aiohttp

scripts/pip-compile.sh

@@ -3,7 +3,16 @@
 # exit when any command fails
 set -e
 
-pip-compile requirements.in $1
+pip-compile \
-pip-compile --output-file=requirements-dev.txt requirements-dev.in $1
+    requirements/requirements.in \
-pip-compile --output-file=requirements-hf-embed.txt requirements-hf-embed.in $1
+    --output-file=requirements.txt \
+    $1
 
+for SUFFIX in dev hf-embed ; do
+    echo suffix: ${SUFFIX}
+    pip-compile \
+        requirements/requirements-${SUFFIX}.in \
+        --output-file=requirements/requirements-${SUFFIX}.txt \
+        $1
+done
+    

setup.py

@@ -1,4 +1,5 @@
 import re
+from pathlib import Path
 
 from setuptools import find_packages, setup
 
@@ -7,15 +8,20 @@ from aider.help_pats import exclude_website_pats
 
 
 def get_requirements(suffix=""):
-    fname = "requirements" + suffix + ".txt"
+    if suffix:
-    with open(fname) as f:
+        fname = "requirements-" + suffix + ".txt"
-        requirements = f.read().splitlines()
+        fname = Path("requirements") / fname
+    else:
+        fname = Path("requirements.txt")
+
+    requirements = fname.read_text().splitlines()
+
     return requirements
 
 
 requirements = get_requirements()
-dev_requirements = get_requirements("-dev")
+dev_requirements = get_requirements("dev")
-hf_requirements = get_requirements("-hf-embed")
+hf_requirements = get_requirements("hf-embed")
 
 # README
 with open("README.md", "r", encoding="utf-8") as f: