diff --git a/requirements-dev.in b/requirements-dev.in index ded94b556..f1d040cd2 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,6 +1,6 @@ -c requirements.txt # -# pip-compile --output-file=dev-requirements.txt dev-requirements.in --upgrade +# pip-compile --output-file=requirements-dev.txt requirements-dev.in --upgrade # pytest pip-tools diff --git a/requirements-dev.txt b/requirements-dev.txt index 54bbd563f..fa026f57d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --output-file=dev-requirements.txt dev-requirements.in +# pip-compile --output-file=requirements-dev.txt requirements-dev.in # alabaster==0.7.16 # via sphinx @@ -26,7 +26,7 @@ click==8.1.7 # pip-tools # typer cogapp==3.4.1 - # via -r dev-requirements.in + # via -r requirements-dev.in contourpy==1.2.1 # via matplotlib cycler==0.12.1 @@ -56,7 +56,7 @@ idna==3.7 imagesize==1.4.1 # via sphinx imgcat==0.5.0 - # via -r dev-requirements.in + # via -r requirements-dev.in iniconfig==2.0.0 # via pytest jinja2==3.1.4 @@ -66,7 +66,7 @@ jinja2==3.1.4 kiwisolver==1.4.5 # via matplotlib lox==0.12.0 - # via -r dev-requirements.in + # via -r requirements-dev.in markdown-it-py==3.0.0 # via # -c requirements.txt @@ -76,7 +76,7 @@ markupsafe==2.1.5 # -c requirements.txt # jinja2 matplotlib==3.9.1 - # via -r dev-requirements.in + # via -r requirements-dev.in mdurl==0.1.2 # via # -c requirements.txt @@ -101,7 +101,7 @@ packaging==24.1 pandas==2.2.2 # via # -c requirements.txt - # -r dev-requirements.in + # -r requirements-dev.in pathos==0.3.2 # via lox pillow==10.4.0 @@ -109,7 +109,7 @@ pillow==10.4.0 # -c requirements.txt # matplotlib pip-tools==7.4.1 - # via -r dev-requirements.in + # via -r requirements-dev.in platformdirs==4.2.2 # via virtualenv pluggy==1.5.0 @@ -119,7 +119,7 @@ pox==0.3.4 ppft==1.7.6.8 # via pathos pre-commit==3.7.1 - # via -r dev-requirements.in + # via -r requirements-dev.in pygments==2.18.0 # via # -c requirements.txt @@ -134,7 +134,7 @@ pyproject-hooks==1.1.0 # build # pip-tools pytest==8.2.2 - # via -r dev-requirements.in + # via -r requirements-dev.in python-dateutil==2.9.0.post0 # via # -c requirements.txt @@ -185,7 +185,7 @@ sphinxcontrib-qthelp==1.0.7 sphinxcontrib-serializinghtml==1.1.10 # via sphinx typer==0.12.3 - # via -r dev-requirements.in + # via -r requirements-dev.in typing-extensions==4.12.2 # via # -c requirements.txt diff --git a/requirements-hf.in b/requirements-hf.in new file mode 100644 index 000000000..51edb71d4 --- /dev/null +++ b/requirements-hf.in @@ -0,0 +1,6 @@ +-c requirements.txt +# +# pip-compile --output-file=requirements-hf.txt requirements-hf.in --upgrade +# + +llama-index-embeddings-huggingface diff --git a/requirements-hf.txt b/requirements-hf.txt new file mode 100644 index 000000000..24c16124d --- /dev/null +++ b/requirements-hf.txt @@ -0,0 +1,312 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --output-file=requirements-hf.txt requirements-hf.in +# +aiohttp==3.9.5 + # via + # -c requirements.txt + # huggingface-hub + # llama-index-core +aiosignal==1.3.1 + # via + # -c requirements.txt + # aiohttp +annotated-types==0.7.0 + # via + # -c requirements.txt + # pydantic +anyio==4.4.0 + # via + # -c requirements.txt + # httpx + # openai +attrs==23.2.0 + # via + # -c requirements.txt + # aiohttp +certifi==2024.7.4 + # via + # -c requirements.txt + # httpcore + # httpx + # requests +charset-normalizer==3.3.2 + # via + # -c requirements.txt + # requests +click==8.1.7 + # via + # -c requirements.txt + # nltk +dataclasses-json==0.6.7 + # via + # -c requirements.txt + # llama-index-core +deprecated==1.2.14 + # via + # -c requirements.txt + # llama-index-core +dirtyjson==1.0.8 + # via + # -c requirements.txt + # llama-index-core +distro==1.9.0 + # via + # -c requirements.txt + # openai +filelock==3.15.4 + # via + # -c requirements.txt + # huggingface-hub + # torch + # transformers +frozenlist==1.4.1 + # via + # -c requirements.txt + # aiohttp + # aiosignal +fsspec==2024.6.1 + # via + # -c requirements.txt + # huggingface-hub + # llama-index-core + # torch +greenlet==3.0.3 + # via + # -c requirements.txt + # sqlalchemy +h11==0.14.0 + # via + # -c requirements.txt + # httpcore +httpcore==1.0.5 + # via + # -c requirements.txt + # httpx +httpx==0.27.0 + # via + # -c requirements.txt + # llama-cloud + # llama-index-core + # openai +huggingface-hub[inference]==0.23.4 + # via + # -c requirements.txt + # llama-index-embeddings-huggingface + # sentence-transformers + # tokenizers + # transformers +idna==3.7 + # via + # -c requirements.txt + # anyio + # httpx + # requests + # yarl +jinja2==3.1.4 + # via + # -c requirements.txt + # torch +joblib==1.4.2 + # via + # -c requirements.txt + # nltk + # scikit-learn +llama-cloud==0.0.6 + # via + # -c requirements.txt + # llama-index-core +llama-index-core==0.10.52.post2 + # via + # -c requirements.txt + # llama-index-embeddings-huggingface +llama-index-embeddings-huggingface==0.2.2 + # via -r requirements-hf.in +markupsafe==2.1.5 + # via + # -c requirements.txt + # jinja2 +marshmallow==3.21.3 + # via + # -c requirements.txt + # dataclasses-json +minijinja==2.0.1 + # via huggingface-hub +mpmath==1.3.0 + # via sympy +multidict==6.0.5 + # via + # -c requirements.txt + # aiohttp + # yarl +mypy-extensions==1.0.0 + # via + # -c requirements.txt + # typing-inspect +nest-asyncio==1.6.0 + # via + # -c requirements.txt + # llama-index-core +networkx==3.2.1 + # via + # -c requirements.txt + # llama-index-core + # torch +nltk==3.8.1 + # via + # -c requirements.txt + # llama-index-core +numpy==1.26.4 + # via + # -c requirements.txt + # llama-index-core + # pandas + # scikit-learn + # scipy + # sentence-transformers + # transformers +openai==1.35.10 + # via + # -c requirements.txt + # llama-index-core +packaging==24.1 + # via + # -c requirements.txt + # huggingface-hub + # marshmallow + # transformers +pandas==2.2.2 + # via + # -c requirements.txt + # llama-index-core +pillow==10.4.0 + # via + # -c requirements.txt + # llama-index-core + # sentence-transformers +pydantic==2.8.2 + # via + # -c requirements.txt + # llama-cloud + # openai +pydantic-core==2.20.1 + # via + # -c requirements.txt + # pydantic +python-dateutil==2.9.0.post0 + # via + # -c requirements.txt + # pandas +pytz==2024.1 + # via + # -c requirements.txt + # pandas +pyyaml==6.0.1 + # via + # -c requirements.txt + # huggingface-hub + # llama-index-core + # transformers +regex==2024.5.15 + # via + # -c requirements.txt + # nltk + # tiktoken + # transformers +requests==2.32.3 + # via + # -c requirements.txt + # huggingface-hub + # llama-index-core + # tiktoken + # transformers +safetensors==0.4.3 + # via transformers +scikit-learn==1.5.1 + # via sentence-transformers +scipy==1.13.1 + # via + # -c requirements.txt + # scikit-learn + # sentence-transformers +sentence-transformers==3.0.1 + # via llama-index-embeddings-huggingface +six==1.16.0 + # via + # -c requirements.txt + # python-dateutil +sniffio==1.3.1 + # via + # -c requirements.txt + # anyio + # httpx + # openai +sqlalchemy[asyncio]==2.0.31 + # via + # -c requirements.txt + # llama-index-core + # sqlalchemy +sympy==1.13.0 + # via torch +tenacity==8.4.2 + # via + # -c requirements.txt + # llama-index-core +threadpoolctl==3.5.0 + # via scikit-learn +tiktoken==0.7.0 + # via + # -c requirements.txt + # llama-index-core +tokenizers==0.19.1 + # via + # -c requirements.txt + # transformers +torch==2.2.2 + # via sentence-transformers +tqdm==4.66.4 + # via + # -c requirements.txt + # huggingface-hub + # llama-index-core + # nltk + # openai + # sentence-transformers + # transformers +transformers==4.42.3 + # via sentence-transformers +typing-extensions==4.12.2 + # via + # -c requirements.txt + # huggingface-hub + # llama-index-core + # openai + # pydantic + # pydantic-core + # sqlalchemy + # torch + # typing-inspect +typing-inspect==0.9.0 + # via + # -c requirements.txt + # dataclasses-json + # llama-index-core +tzdata==2024.1 + # via + # -c requirements.txt + # pandas +urllib3==2.2.2 + # via + # -c requirements.txt + # requests +wrapt==1.16.0 + # via + # -c requirements.txt + # deprecated + # llama-index-core +yarl==1.9.4 + # via + # -c requirements.txt + # aiohttp diff --git a/requirements.in b/requirements.in index 21e6ef197..169eeb698 100644 --- a/requirements.in +++ b/requirements.in @@ -2,9 +2,6 @@ # pip-compile requirements.in --upgrade # -# To install with the CPU version of torch, because the GPU versions are huge ----extra-index-url https://download.pytorch.org/whl/cpu - configargparse GitPython openai @@ -32,7 +29,6 @@ streamlit watchdog flake8 llama-index-core -llama-index-embeddings-huggingface importlib_resources # v3.3 no longer works on python 3.9 diff --git a/requirements.txt b/requirements.txt index ec5d86099..18ec70d3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ # aiohttp==3.9.5 # via - # huggingface-hub # litellm # llama-index-core aiosignal==1.3.1 @@ -65,10 +64,7 @@ diskcache==5.6.3 distro==1.9.0 # via openai filelock==3.15.4 - # via - # huggingface-hub - # torch - # transformers + # via huggingface-hub flake8==7.1.0 # via -r requirements.in frozenlist==1.4.1 @@ -79,7 +75,6 @@ fsspec==2024.6.1 # via # huggingface-hub # llama-index-core - # torch gitdb==4.0.11 # via gitpython gitpython==3.1.43 @@ -135,12 +130,8 @@ httpx==0.27.0 # llama-cloud # llama-index-core # openai -huggingface-hub[inference]==0.23.4 - # via - # llama-index-embeddings-huggingface - # sentence-transformers - # tokenizers - # transformers +huggingface-hub==0.23.4 + # via tokenizers idna==3.7 # via # anyio @@ -158,11 +149,8 @@ jinja2==3.1.4 # altair # litellm # pydeck - # torch joblib==1.4.2 - # via - # nltk - # scikit-learn + # via nltk jsonschema==4.22.0 # via # -r requirements.in @@ -175,10 +163,6 @@ litellm==1.41.6 llama-cloud==0.0.6 # via llama-index-core llama-index-core==0.10.52.post2 - # via - # -r requirements.in - # llama-index-embeddings-huggingface -llama-index-embeddings-huggingface==0.2.2 # via -r requirements.in markdown-it-py==3.0.0 # via rich @@ -190,10 +174,6 @@ mccabe==0.7.0 # via flake8 mdurl==0.1.2 # via markdown-it-py -minijinja==2.0.1 - # via huggingface-hub -mpmath==1.3.0 - # via sympy multidict==6.0.5 # via # aiohttp @@ -206,7 +186,6 @@ networkx==3.2.1 # via # -r requirements.in # llama-index-core - # torch nltk==3.8.1 # via llama-index-core numpy==1.26.4 @@ -217,11 +196,8 @@ numpy==1.26.4 # pandas # pyarrow # pydeck - # scikit-learn # scipy - # sentence-transformers # streamlit - # transformers openai==1.35.10 # via # -r requirements.in @@ -234,7 +210,6 @@ packaging==24.1 # huggingface-hub # marshmallow # streamlit - # transformers pandas==2.2.2 # via # altair @@ -248,7 +223,6 @@ pillow==10.4.0 # via # -r requirements.in # llama-index-core - # sentence-transformers # streamlit playwright==1.45.0 # via -r requirements.in @@ -310,7 +284,6 @@ pyyaml==6.0.1 # -r requirements.in # huggingface-hub # llama-index-core - # transformers referencing==0.35.1 # via # jsonschema @@ -319,7 +292,6 @@ regex==2024.5.15 # via # nltk # tiktoken - # transformers requests==2.32.3 # via # google-api-core @@ -328,7 +300,6 @@ requests==2.32.3 # llama-index-core # streamlit # tiktoken - # transformers rich==13.7.1 # via # -r requirements.in @@ -339,17 +310,8 @@ rpds-py==0.18.1 # referencing rsa==4.9 # via google-auth -safetensors==0.4.3 - # via transformers -scikit-learn==1.5.1 - # via sentence-transformers scipy==1.13.1 - # via - # -r requirements.in - # scikit-learn - # sentence-transformers -sentence-transformers==3.0.1 - # via llama-index-embeddings-huggingface + # via -r requirements.in six==1.16.0 # via python-dateutil smmap==5.0.1 @@ -371,29 +333,21 @@ sqlalchemy[asyncio]==2.0.31 # sqlalchemy streamlit==1.36.0 # via -r requirements.in -sympy==1.12.1 - # via torch tenacity==8.4.2 # via # llama-index-core # streamlit -threadpoolctl==3.5.0 - # via scikit-learn tiktoken==0.7.0 # via # -r requirements.in # litellm # llama-index-core tokenizers==0.19.1 - # via - # litellm - # transformers + # via litellm toml==0.10.2 # via streamlit toolz==0.12.1 # via altair -torch==2.2.2 - # via sentence-transformers tornado==6.4.1 # via streamlit tqdm==4.66.4 @@ -403,10 +357,6 @@ tqdm==4.66.4 # llama-index-core # nltk # openai - # sentence-transformers - # transformers -transformers==4.42.3 - # via sentence-transformers tree-sitter==0.21.3 # via # -r requirements.in @@ -424,7 +374,6 @@ typing-extensions==4.12.2 # pyee # sqlalchemy # streamlit - # torch # typing-inspect typing-inspect==0.9.0 # via diff --git a/scripts/pip-compile.sh b/scripts/pip-compile.sh new file mode 100755 index 000000000..9c9f81f87 --- /dev/null +++ b/scripts/pip-compile.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# exit when any command fails +set -e + +pip-compile requirements.in $1 +pip-compile --output-file=requirements-dev.txt requirements-dev.in $1 +pip-compile --output-file=requirements-hf.txt requirements-hf.in $1 +