feat: create bash library to handle install/run/test of python backends (#2286)

* feat: create bash library to handle install/run/test of python backends

Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com>

* chore: minor cleanup

Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com>

* fix: remove incorrect LIMIT_TARGETS from parler-tts

Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com>

* fix: update runUnitests to handle running tests from a custom test file

Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com>

* chore: document runUnittests

Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com>

---------

Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com>

backend/python/petals/Makefile

@@ -28,4 +28,4 @@ backend_pb2_grpc.py backend_pb2.py:
 
 .PHONY: clean
 clean: protogen-clean
-	rm -rf venv
+	rm -rf venv __pycache__

backend/python/petals/install.sh

@@ -1,34 +1,6 @@
 #!/bin/bash
-set -ex
+set -e
 
-BUILD_ISOLATION_FLAG=""
+source $(dirname $0)/../common/libbackend.sh
 
-MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
-
-uv venv ${MY_DIR}/venv
-source ${MY_DIR}/venv/bin/activate
-
-if [ -f "requirements-install.txt" ]; then
-    # If we have a requirements-install.txt, it means that a package does not properly declare it's build time
-    # dependencies per PEP-517, so we have to set up the proper build environment ourselves, and then install
-    # the package without build isolation
-    BUILD_ISOLATION_FLAG="--no-build-isolation"
-    uv pip install --requirement ${MY_DIR}/requirements-install.txt
-fi
-uv pip install ${BUILD_ISOLATION_FLAG} --requirement ${MY_DIR}/requirements.txt
-
-if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
-    uv pip install ${BUILD_ISOLATION_FLAG}  --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
-fi
-
-if [ -d "/opt/intel" ]; then
-    # Intel GPU: If the directory exists, we assume we are using the Intel image
-    # https://github.com/intel/intel-extension-for-pytorch/issues/538
-    if [ -f "requirements-intel.txt" ]; then
-        uv pip install ${BUILD_ISOLATION_FLAG}  --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
-    fi
-fi
-
-if [ "$PIP_CACHE_PURGE" = true ] ; then
-    pip cache purge
-fi
+installRequirements

backend/python/petals/petals.yml

@@ -1,30 +0,0 @@
-name: petals
-channels:
-  - defaults
-dependencies:
-  # - _libgcc_mutex=0.1=main
-  # - _openmp_mutex=5.1=1_gnu
-  # - bzip2=1.0.8=h7b6447c_0
-  # - ca-certificates=2023.08.22=h06a4308_0
-  # - ld_impl_linux-64=2.38=h1181459_1
-  # - libffi=3.4.4=h6a678d5_0
-  # - libgcc-ng=11.2.0=h1234567_1
-  # - libgomp=11.2.0=h1234567_1
-  # - libstdcxx-ng=11.2.0=h1234567_1
-  # - libuuid=1.41.5=h5eee18b_0
-  # - ncurses=6.4=h6a678d5_0
-  # - openssl=3.0.11=h7f8727e_2
-  # - pip=23.2.1=py311h06a4308_0
-  - python=3.11.5=h955ad1f_0
-  # - readline=8.2=h5eee18b_0
-  # - setuptools=68.0.0=py311h06a4308_0
-  # - sqlite=3.41.2=h5eee18b_0
-  # - tk=8.6.12=h1ccaba5_0
-  # - tzdata=2023c=h04d1e81_0
-  # - wheel=0.41.2=py311h06a4308_0
-  # - xz=5.4.2=h5eee18b_0
-  # - zlib=1.2.13=h5eee18b_0
-  - pip:
-      - torch==2.1.0
-      - git+https://github.com/bigscience-workshop/petals
-prefix: /opt/conda/envs/petals

backend/python/petals/run.sh

@@ -1,10 +1,4 @@
 #!/bin/bash
+source $(dirname $0)/../common/libbackend.sh
 
-##
-## A bash script wrapper that runs the GRPC backend
-
-MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
-
-source $MY_DIR/venv/bin/activate
-
-python $MY_DIR/backend_petals.py $@
+startBackend $@

backend/python/petals/test.py

@@ -20,7 +20,7 @@ class TestBackendServicer(unittest.TestCase):
     This class contains methods to test the startup and shutdown of the gRPC service.
     """
     def setUp(self):
-        self.service = subprocess.Popen(["python", "backend_petals.py", "--addr", "localhost:50051"])
+        self.service = subprocess.Popen(["python", "backend.py", "--addr", "localhost:50051"])
         time.sleep(10)
 
     def tearDown(self) -> None:

backend/python/petals/test.sh

@@ -1,16 +1,6 @@
 #!/bin/bash
-##
-## A bash script wrapper that runs python unittests
+set -e
 
-MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+source $(dirname $0)/../common/libbackend.sh
 
-source $MY_DIR/venv/bin/activate
-
-if [ -f "${MY_DIR}/test.py" ]; then
-    pushd ${MY_DIR}
-    python -m unittest test.py
-    popd
-else
-    echo "ERROR: No tests defined for backend!"
-    exit 1
-fi
+runUnittests