feat: create bash library to handle install/run/test of python backends (#2286)

* feat: create bash library to handle install/run/test of python backends Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * chore: minor cleanup Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: remove incorrect LIMIT_TARGETS from parler-tts Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: update runUnitests to handle running tests from a custom test file Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * chore: document runUnittests Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> --------- Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com>
2025-05-20 18:45:00 +00:00 · 2024-05-11 11:32:46 -05:00 · 2024-05-11 11:32:46 -05:00 · e2de8a88f7
commit e2de8a88f7
parent 7f4febd6c2
106 changed files with 425 additions and 1606 deletions
--- a/backend/python/vall-e-x/Makefile
+++ b/backend/python/vall-e-x/Makefile
@ -30,4 +30,4 @@ backend_pb2_grpc.py backend_pb2.py:

 .PHONY: clean
 clean: protogen-clean
-	rm -rf source venv
+	rm -rf source venv __pycache__
--- a/backend/python/vall-e-x/ttsvalle.py
+++ b/backend/python/vall-e-x/ttsvalle.py
--- a/backend/python/vall-e-x/install.sh
+++ b/backend/python/vall-e-x/install.sh
@ -1,40 +1,14 @@
 #!/bin/bash
-set -ex
+set -e

-BUILD_ISOLATION_FLAG=""
+VALL_E_X_VERSION=3faaf8ccadb154d63b38070caf518ce9309ea0f4

-MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+source $(dirname $0)/../common/libbackend.sh

-uv venv ${MY_DIR}/venv
-source ${MY_DIR}/venv/bin/activate
+installRequirements

-if [ -f "requirements-install.txt" ]; then
-    # If we have a requirements-install.txt, it means that a package does not properly declare it's build time
-    # dependencies per PEP-517, so we have to set up the proper build environment ourselves, and then install
-    # the package without build isolation
-    BUILD_ISOLATION_FLAG="--no-build-isolation"
-    uv pip install --requirement ${MY_DIR}/requirements-install.txt
-fi
-uv pip install ${BUILD_ISOLATION_FLAG} --requirement ${MY_DIR}/requirements.txt
-
-if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
-    uv pip install ${BUILD_ISOLATION_FLAG}  --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
-fi
-
-if [ -d "/opt/intel" ]; then
-    # Intel GPU: If the directory exists, we assume we are using the Intel image
-    # https://github.com/intel/intel-extension-for-pytorch/issues/538
-    if [ -f "requirements-intel.txt" ]; then
-        uv pip install ${BUILD_ISOLATION_FLAG}  --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
-    fi
-fi
-
-git clone https://github.com/Plachtaa/VALL-E-X.git $MY_DIR/source
-pushd $MY_DIR/source && git checkout -b build $VALL_E_X_VERSION && popd
+git clone https://github.com/Plachtaa/VALL-E-X.git ${MY_DIR}/source
+pushd ${MY_DIR}/source && git checkout -b build ${VALL_E_X_VERSION} && popd
 uv pip install ${BUILD_ISOLATION_FLAG} --requirement ${MY_DIR}/source/requirements.txt

-cp -rfv ./*py $MY_DIR/source/
-
-if [ "$PIP_CACHE_PURGE" = true ] ; then
-    pip cache purge
-fi
+cp -v ./*py $MY_DIR/source/
--- a/backend/python/vall-e-x/run.sh
+++ b/backend/python/vall-e-x/run.sh
@ -1,10 +1,6 @@
 #!/bin/bash
+BACKEND_FILE="${MY_DIR}/source/backend.py"

-##
-## A bash script wrapper that runs the GRPC backend
+source $(dirname $0)/../common/libbackend.sh

-MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
-
-source $MY_DIR/venv/bin/activate
-
-pushd $MY_DIR/source && python ttsvalle.py $@
+startBackend $@
--- a/backend/python/vall-e-x/test.py
+++ b/backend/python/vall-e-x/test.py
@ -18,7 +18,7 @@ class TestBackendServicer(unittest.TestCase):
        """
        This method sets up the gRPC service by starting the server
        """
-        self.service = subprocess.Popen(["python3", "ttsvalle.py", "--addr", "localhost:50051"])
+        self.service = subprocess.Popen(["python3", "backend.py", "--addr", "localhost:50051"])
        time.sleep(10)

    def tearDown(self) -> None:
--- a/backend/python/vall-e-x/test.sh
+++ b/backend/python/vall-e-x/test.sh
@ -1,16 +1,7 @@
 #!/bin/bash
-##
-## A bash script wrapper that runs python unittests
+set -e
+TEST_FILE="./source/test.py"

-MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+source $(dirname $0)/../common/libbackend.sh

-source $MY_DIR/venv/bin/activate
-
-if [ -f "${MY_DIR}/test.py" ]; then
-    pushd ${MY_DIR}/source
-    python -m unittest test.py
-    popd
-else
-    echo "ERROR: No tests defined for backend!"
-    exit 1
-fi
+runUnittests
--- a/backend/python/vall-e-x/ttsvalle.yml
+++ b/backend/python/vall-e-x/ttsvalle.yml
@ -1,101 +0,0 @@
-name: ttsvalle
-channels:
-  - defaults
-dependencies:
-  - _libgcc_mutex=0.1=main
-  - _openmp_mutex=5.1=1_gnu
-  - bzip2=1.0.8=h7b6447c_0
-  - ca-certificates=2023.08.22=h06a4308_0
-  - ld_impl_linux-64=2.38=h1181459_1
-  - libffi=3.4.4=h6a678d5_0
-  - libgcc-ng=11.2.0=h1234567_1
-  - libgomp=11.2.0=h1234567_1
-  - libstdcxx-ng=11.2.0=h1234567_1
-  - libuuid=1.41.5=h5eee18b_0
-  - ncurses=6.4=h6a678d5_0
-  - openssl=3.0.11=h7f8727e_2
-  - pip=23.2.1=py310h06a4308_0
-  - python=3.10.13=h955ad1f_0
-  - readline=8.2=h5eee18b_0
-  - setuptools=68.0.0=py310h06a4308_0
-  - sqlite=3.41.2=h5eee18b_0
-  - tk=8.6.12=h1ccaba5_0
-  - tzdata=2023c=h04d1e81_0
-  - wheel=0.41.2=py310h06a4308_0
-  - xz=5.4.2=h5eee18b_0
-  - zlib=1.2.13=h5eee18b_0
-  - pip:
-      - aiofiles==23.2.1
-      - altair==5.1.2
-      - annotated-types==0.6.0
-      - anyio==3.7.1
-      - click==8.1.7
-      - cn2an==0.5.22
-      - cython==3.0.3
-      - einops==0.7.0
-      - encodec==0.1.1
-      - eng-to-ipa==0.0.2
-      - fastapi==0.103.2
-      - ffmpeg-python==0.2.0
-      - ffmpy==0.3.1
-      - fsspec==2023.9.2
-      - future==0.18.3
-      - gradio==3.47.1
-      - gradio-client==0.6.0
-      - grpcio==1.63.0
-      - h11==0.14.0
-      - httpcore==0.18.0
-      - httpx==0.25.0
-      - huggingface-hub==0.17.3
-      - importlib-resources==6.1.0
-      - inflect==7.0.0
-      - jieba==0.42.1
-      - langid==1.1.6
-      - llvmlite==0.41.0
-      - more-itertools==10.1.0
-      - nltk==3.8.1
-      - numba==0.58.0
-      - numpy==1.25.2
-      - nvidia-cublas-cu12==12.1.3.1
-      - nvidia-cuda-cupti-cu12==12.1.105
-      - nvidia-cuda-nvrtc-cu12==12.1.105
-      - nvidia-cuda-runtime-cu12==12.1.105
-      - nvidia-cudnn-cu12==8.9.2.26
-      - nvidia-cufft-cu12==11.0.2.54
-      - nvidia-curand-cu12==10.3.2.106
-      - nvidia-cusolver-cu12==11.4.5.107
-      - nvidia-cusparse-cu12==12.1.0.106
-      - nvidia-nccl-cu12==2.18.1
-      - nvidia-nvjitlink-cu12==12.2.140
-      - nvidia-nvtx-cu12==12.1.105
-      - openai-whisper==20230306
-      - orjson==3.9.7
-      - proces==0.1.7
-      - protobuf==4.24.4
-      - pydantic==2.4.2
-      - pydantic-core==2.10.1
-      - pydub==0.25.1
-      - pyopenjtalk-prebuilt==0.3.0
-      - pypinyin==0.49.0
-      - python-multipart==0.0.6
-      - regex==2023.10.3
-      - safetensors>=0.4.0
-      - semantic-version==2.10.0
-      - soundfile==0.12.1
-      - starlette==0.27.0
-      - sudachidict-core==20230927
-      - sudachipy==0.6.7
-      - tokenizers==0.14.1
-      - toolz==0.12.0
-      - torch==2.1.0
-      - torchaudio==2.1.0
-      - torchvision==0.16.0
-      - tqdm==4.66.1
-      - transformers==4.34.0
-      - triton==2.1.0
-      - unidecode==1.3.7
-      - uvicorn==0.23.2
-      - vocos==0.0.3
-      - websockets==11.0.3
-      - wget==3.2
-prefix: /opt/conda/envs/ttsvalle