refactor reqs into requirements/

This commit is contained in:
Paul Gauthier 2024-07-13 16:36:41 +01:00
parent 82a59416d4
commit bafa8ab71c
8 changed files with 137 additions and 122 deletions

View file

@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile with Python 3.12
# by the following command:
#
# pip-compile requirements.in
# pip-compile --output-file=requirements.txt requirements/requirements.in
#
aiohttp==3.9.5
# via litellm
@ -22,9 +22,9 @@ attrs==23.2.0
# jsonschema
# referencing
backoff==2.2.1
# via -r requirements.in
# via -r requirements/requirements.in
beautifulsoup4==4.12.3
# via -r requirements.in
# via -r requirements/requirements.in
blinker==1.8.2
# via streamlit
cachetools==5.3.3
@ -47,17 +47,17 @@ click==8.1.7
# litellm
# streamlit
configargparse==1.7
# via -r requirements.in
# via -r requirements/requirements.in
diff-match-patch==20230430
# via -r requirements.in
# via -r requirements/requirements.in
diskcache==5.6.3
# via -r requirements.in
# via -r requirements/requirements.in
distro==1.9.0
# via openai
filelock==3.15.4
# via huggingface-hub
flake8==7.1.0
# via -r requirements.in
# via -r requirements/requirements.in
frozenlist==1.4.1
# via
# aiohttp
@ -68,7 +68,7 @@ gitdb==4.0.11
# via gitpython
gitpython==3.1.43
# via
# -r requirements.in
# -r requirements/requirements.in
# streamlit
google-ai-generativelanguage==0.6.6
# via google-generativeai
@ -89,7 +89,7 @@ google-auth==2.31.0
google-auth-httplib2==0.2.0
# via google-api-python-client
google-generativeai==0.7.1
# via -r requirements.in
# via -r requirements/requirements.in
googleapis-common-protos==1.63.2
# via
# google-api-core
@ -97,7 +97,7 @@ googleapis-common-protos==1.63.2
greenlet==3.0.3
# via playwright
grep-ast==0.3.2
# via -r requirements.in
# via -r requirements/requirements.in
grpcio==1.64.1
# via
# google-api-core
@ -124,10 +124,10 @@ idna==3.7
# yarl
importlib-metadata==7.2.1
# via
# -r requirements.in
# -r requirements/requirements.in
# litellm
importlib-resources==6.4.0
# via -r requirements.in
# via -r requirements/requirements.in
jinja2==3.1.4
# via
# altair
@ -135,13 +135,13 @@ jinja2==3.1.4
# pydeck
jsonschema==4.22.0
# via
# -r requirements.in
# -r requirements/requirements.in
# altair
# litellm
jsonschema-specifications==2023.12.1
# via jsonschema
litellm==1.41.6
# via -r requirements.in
# via -r requirements/requirements.in
markdown-it-py==3.0.0
# via rich
markupsafe==2.1.5
@ -155,10 +155,10 @@ multidict==6.0.5
# aiohttp
# yarl
networkx==3.2.1
# via -r requirements.in
# via -r requirements/requirements.in
numpy==1.26.4
# via
# -r requirements.in
# -r requirements/requirements.in
# altair
# pandas
# pyarrow
@ -169,7 +169,7 @@ openai==1.35.10
# via litellm
packaging==24.1
# via
# -r requirements.in
# -r requirements/requirements.in
# altair
# huggingface-hub
# streamlit
@ -179,16 +179,16 @@ pandas==2.2.2
# streamlit
pathspec==0.12.1
# via
# -r requirements.in
# -r requirements/requirements.in
# grep-ast
pillow==10.4.0
# via
# -r requirements.in
# -r requirements/requirements.in
# streamlit
playwright==1.45.0
# via -r requirements.in
# via -r requirements/requirements.in
prompt-toolkit==3.0.47
# via -r requirements.in
# via -r requirements/requirements.in
proto-plus==1.24.0
# via
# google-ai-generativelanguage
@ -230,7 +230,7 @@ pyflakes==3.2.0
pygments==2.18.0
# via rich
pypandoc==1.13
# via -r requirements.in
# via -r requirements/requirements.in
pyparsing==3.1.2
# via httplib2
python-dateutil==2.9.0.post0
@ -241,7 +241,7 @@ pytz==2024.1
# via pandas
pyyaml==6.0.1
# via
# -r requirements.in
# -r requirements/requirements.in
# huggingface-hub
referencing==0.35.1
# via
@ -258,7 +258,7 @@ requests==2.32.3
# tiktoken
rich==13.7.1
# via
# -r requirements.in
# -r requirements/requirements.in
# streamlit
rpds-py==0.18.1
# via
@ -267,7 +267,7 @@ rpds-py==0.18.1
rsa==4.9
# via google-auth
scipy==1.13.1
# via -r requirements.in
# via -r requirements/requirements.in
six==1.16.0
# via python-dateutil
smmap==5.0.1
@ -278,13 +278,13 @@ sniffio==1.3.1
# httpx
# openai
sounddevice==0.4.7
# via -r requirements.in
# via -r requirements/requirements.in
soundfile==0.12.1
# via -r requirements.in
# via -r requirements/requirements.in
soupsieve==2.5
# via beautifulsoup4
streamlit==1.36.0
# via -r requirements.in
# via -r requirements/requirements.in
tenacity==8.4.2
# via streamlit
tiktoken==0.7.0
@ -304,7 +304,7 @@ tqdm==4.66.4
# openai
tree-sitter==0.21.3
# via
# -r requirements.in
# -r requirements/requirements.in
# tree-sitter-languages
tree-sitter-languages==1.10.2
# via grep-ast
@ -324,7 +324,7 @@ uritemplate==4.1.1
urllib3==2.2.2
# via requests
watchdog==4.0.1
# via -r requirements.in
# via -r requirements/requirements.in
wcwidth==0.2.13
# via prompt-toolkit
yarl==1.9.4

View file

@ -1,4 +1,4 @@
-c requirements.txt
-c ../requirements.txt
#
# pip-compile --output-file=requirements-dev.txt requirements-dev.in --upgrade
#

View file

@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile with Python 3.12
# by the following command:
#
# pip-compile --output-file=requirements-dev.txt requirements-dev.in
# pip-compile --output-file=requirements/requirements-dev.txt requirements/requirements-dev.in
#
alabaster==0.7.16
# via sphinx
@ -12,21 +12,21 @@ build==1.2.1
# via pip-tools
certifi==2024.7.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# requests
cfgv==3.4.0
# via pre-commit
charset-normalizer==3.3.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# requests
click==8.1.7
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# pip-tools
# typer
cogapp==3.4.1
# via -r requirements-dev.in
# via -r requirements/requirements-dev.in
contourpy==1.2.1
# via matplotlib
cycler==0.12.1
@ -43,7 +43,7 @@ docutils==0.20.1
# sphinx-rtd-theme
filelock==3.15.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# virtualenv
fonttools==4.53.1
# via matplotlib
@ -51,35 +51,35 @@ identify==2.5.36
# via pre-commit
idna==3.7
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# requests
imagesize==1.4.1
# via sphinx
imgcat==0.5.0
# via -r requirements-dev.in
# via -r requirements/requirements-dev.in
iniconfig==2.0.0
# via pytest
jinja2==3.1.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# sphinx
kiwisolver==1.4.5
# via matplotlib
lox==0.12.0
# via -r requirements-dev.in
# via -r requirements/requirements-dev.in
markdown-it-py==3.0.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# rich
markupsafe==2.1.5
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# jinja2
matplotlib==3.9.1
# via -r requirements-dev.in
# via -r requirements/requirements-dev.in
mdurl==0.1.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# markdown-it-py
multiprocess==0.70.16
# via pathos
@ -87,29 +87,29 @@ nodeenv==1.9.1
# via pre-commit
numpy==1.26.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# contourpy
# matplotlib
# pandas
packaging==24.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# build
# matplotlib
# pytest
# sphinx
pandas==2.2.2
# via
# -c requirements.txt
# -r requirements-dev.in
# -c requirements/../requirements.txt
# -r requirements/requirements-dev.in
pathos==0.3.2
# via lox
pillow==10.4.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# matplotlib
pip-tools==7.4.1
# via -r requirements-dev.in
# via -r requirements/requirements-dev.in
platformdirs==4.2.2
# via virtualenv
pluggy==1.5.0
@ -119,48 +119,48 @@ pox==0.3.4
ppft==1.7.6.8
# via pathos
pre-commit==3.7.1
# via -r requirements-dev.in
# via -r requirements/requirements-dev.in
pygments==2.18.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# rich
# sphinx
pyparsing==3.1.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# matplotlib
pyproject-hooks==1.1.0
# via
# build
# pip-tools
pytest==8.2.2
# via -r requirements-dev.in
# via -r requirements/requirements-dev.in
python-dateutil==2.9.0.post0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# matplotlib
# pandas
pytz==2024.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# pandas
pyyaml==6.0.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# pre-commit
requests==2.32.3
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# sphinx
rich==13.7.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# typer
shellingham==1.5.4
# via typer
six==1.16.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# python-dateutil
snowballstemmer==2.2.0
# via sphinx
@ -185,18 +185,18 @@ sphinxcontrib-qthelp==1.0.7
sphinxcontrib-serializinghtml==1.1.10
# via sphinx
typer==0.12.3
# via -r requirements-dev.in
# via -r requirements/requirements-dev.in
typing-extensions==4.12.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# typer
tzdata==2024.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# pandas
urllib3==2.2.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# requests
virtualenv==20.26.3
# via pre-commit

View file

@ -1,4 +1,4 @@
-c requirements.txt
-c ../requirements.txt
#
# pip-compile --output-file=requirements-hf.txt requirements-hf.in --upgrade
#

View file

@ -2,43 +2,43 @@
# This file is autogenerated by pip-compile with Python 3.12
# by the following command:
#
# pip-compile --output-file=requirements-hf-embed.txt requirements-hf-embed.in
# pip-compile --output-file=requirements/requirements-hf-embed.txt requirements/requirements-hf-embed.in
#
aiohttp==3.9.5
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# huggingface-hub
# llama-index-core
aiosignal==1.3.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# aiohttp
annotated-types==0.7.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# pydantic
anyio==4.4.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# httpx
# openai
attrs==23.2.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# aiohttp
certifi==2024.7.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# httpcore
# httpx
# requests
charset-normalizer==3.3.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# requests
click==8.1.7
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# nltk
dataclasses-json==0.6.7
# via llama-index-core
@ -48,60 +48,60 @@ dirtyjson==1.0.8
# via llama-index-core
distro==1.9.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# openai
filelock==3.15.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# huggingface-hub
# torch
# transformers
frozenlist==1.4.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# aiohttp
# aiosignal
fsspec==2024.6.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# huggingface-hub
# llama-index-core
# torch
greenlet==3.0.3
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# sqlalchemy
h11==0.14.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# httpcore
httpcore==1.0.5
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# httpx
httpx==0.27.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-cloud
# llama-index-core
# openai
huggingface-hub[inference]==0.23.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-index-embeddings-huggingface
# sentence-transformers
# tokenizers
# transformers
idna==3.7
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# anyio
# httpx
# requests
# yarl
jinja2==3.1.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# torch
joblib==1.4.2
# via
@ -111,13 +111,13 @@ llama-cloud==0.0.6
# via llama-index-core
llama-index-core==0.10.52.post2
# via
# -r requirements-hf-embed.in
# -r requirements/requirements-hf-embed.in
# llama-index-embeddings-huggingface
llama-index-embeddings-huggingface==0.2.2
# via -r requirements-hf-embed.in
# via -r requirements/requirements-hf-embed.in
markupsafe==2.1.5
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# jinja2
marshmallow==3.21.3
# via dataclasses-json
@ -127,7 +127,7 @@ mpmath==1.3.0
# via sympy
multidict==6.0.5
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# aiohttp
# yarl
mypy-extensions==1.0.0
@ -136,14 +136,14 @@ nest-asyncio==1.6.0
# via llama-index-core
networkx==3.2.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-index-core
# torch
nltk==3.8.1
# via llama-index-core
numpy==1.26.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-index-core
# pandas
# scikit-learn
@ -152,55 +152,55 @@ numpy==1.26.4
# transformers
openai==1.35.10
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-index-core
packaging==24.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# huggingface-hub
# marshmallow
# transformers
pandas==2.2.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-index-core
pillow==10.4.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-index-core
# sentence-transformers
pydantic==2.8.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-cloud
# openai
pydantic-core==2.20.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# pydantic
python-dateutil==2.9.0.post0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# pandas
pytz==2024.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# pandas
pyyaml==6.0.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# huggingface-hub
# llama-index-core
# transformers
regex==2024.5.15
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# nltk
# tiktoken
# transformers
requests==2.32.3
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# huggingface-hub
# llama-index-core
# tiktoken
@ -211,18 +211,18 @@ scikit-learn==1.5.1
# via sentence-transformers
scipy==1.13.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# scikit-learn
# sentence-transformers
sentence-transformers==3.0.1
# via llama-index-embeddings-huggingface
six==1.16.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# python-dateutil
sniffio==1.3.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# anyio
# httpx
# openai
@ -234,23 +234,23 @@ sympy==1.13.0
# via torch
tenacity==8.4.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-index-core
threadpoolctl==3.5.0
# via scikit-learn
tiktoken==0.7.0
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# llama-index-core
tokenizers==0.19.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# transformers
torch==2.2.2
# via sentence-transformers
tqdm==4.66.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# huggingface-hub
# llama-index-core
# nltk
@ -261,7 +261,7 @@ transformers==4.42.3
# via sentence-transformers
typing-extensions==4.12.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# huggingface-hub
# llama-index-core
# openai
@ -276,11 +276,11 @@ typing-inspect==0.9.0
# llama-index-core
tzdata==2024.1
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# pandas
urllib3==2.2.2
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# requests
wrapt==1.16.0
# via
@ -288,5 +288,5 @@ wrapt==1.16.0
# llama-index-core
yarl==1.9.4
# via
# -c requirements.txt
# -c requirements/../requirements.txt
# aiohttp

View file

@ -3,7 +3,16 @@
# exit when any command fails
set -e
pip-compile requirements.in $1
pip-compile --output-file=requirements-dev.txt requirements-dev.in $1
pip-compile --output-file=requirements-hf-embed.txt requirements-hf-embed.in $1
pip-compile \
requirements/requirements.in \
--output-file=requirements.txt \
$1
for SUFFIX in dev hf-embed ; do
echo suffix: ${SUFFIX}
pip-compile \
requirements/requirements-${SUFFIX}.in \
--output-file=requirements/requirements-${SUFFIX}.txt \
$1
done

View file

@ -1,4 +1,5 @@
import re
from pathlib import Path
from setuptools import find_packages, setup
@ -7,15 +8,20 @@ from aider.help_pats import exclude_website_pats
def get_requirements(suffix=""):
fname = "requirements" + suffix + ".txt"
with open(fname) as f:
requirements = f.read().splitlines()
if suffix:
fname = "requirements-" + suffix + ".txt"
fname = Path("requirements") / fname
else:
fname = Path("requirements.txt")
requirements = fname.read_text().splitlines()
return requirements
requirements = get_requirements()
dev_requirements = get_requirements("-dev")
hf_requirements = get_requirements("-hf-embed")
dev_requirements = get_requirements("dev")
hf_requirements = get_requirements("hf-embed")
# README
with open("README.md", "r", encoding="utf-8") as f: