From d35f408702ab4679e60ca6c2f745e493433d2a1c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 19 Dec 2023 23:34:59 +0000 Subject: [PATCH] feat(conda): share env between diffusers and bark --- backend/python/bark/Makefile | 4 +- backend/python/bark/run.sh | 2 +- backend/python/bark/test.sh | 2 +- .../python/common-env/transformers/Makefile | 5 ++ .../transformers/transformers.yml} | 4 +- backend/python/diffusers/Makefile | 4 +- backend/python/diffusers/diffusers.yml | 73 ------------------- backend/python/diffusers/run.sh | 2 +- backend/python/diffusers/test.sh | 2 +- 9 files changed, 14 insertions(+), 84 deletions(-) create mode 100644 backend/python/common-env/transformers/Makefile rename backend/python/{bark/ttsbark.yml => common-env/transformers/transformers.yml} (97%) delete mode 100644 backend/python/diffusers/diffusers.yml diff --git a/backend/python/bark/Makefile b/backend/python/bark/Makefile index f55c645d..68f73b29 100644 --- a/backend/python/bark/Makefile +++ b/backend/python/bark/Makefile @@ -1,8 +1,6 @@ .PHONY: ttsbark ttsbark: - @echo "Creating virtual environment..." - @conda env create --name ttsbark --file ttsbark.yml - @echo "Virtual environment created." + $(MAKE) -C ../common-env/transformers .PHONY: run run: diff --git a/backend/python/bark/run.sh b/backend/python/bark/run.sh index 63e62cd7..7e6a36bb 100755 --- a/backend/python/bark/run.sh +++ b/backend/python/bark/run.sh @@ -6,7 +6,7 @@ export PATH=$PATH:/opt/conda/bin # Activate conda environment -source activate ttsbark +source activate transformers # get the directory where the bash script is located DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" diff --git a/backend/python/bark/test.sh b/backend/python/bark/test.sh index 33fd0c5a..17671068 100644 --- a/backend/python/bark/test.sh +++ b/backend/python/bark/test.sh @@ -3,7 +3,7 @@ ## A bash script wrapper that runs the bark server with conda # Activate conda environment -source activate ttsbark +source activate transformers # get the directory where the bash script is located DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" diff --git a/backend/python/common-env/transformers/Makefile b/backend/python/common-env/transformers/Makefile new file mode 100644 index 00000000..b0dbb18b --- /dev/null +++ b/backend/python/common-env/transformers/Makefile @@ -0,0 +1,5 @@ +.PHONY: transformers +transformers: + @echo "Creating virtual environment..." + @conda env create --name transformers --file transformers.yml + @echo "Virtual environment created." \ No newline at end of file diff --git a/backend/python/bark/ttsbark.yml b/backend/python/common-env/transformers/transformers.yml similarity index 97% rename from backend/python/bark/ttsbark.yml rename to backend/python/common-env/transformers/transformers.yml index bbeb4934..fed2bce7 100644 --- a/backend/python/bark/ttsbark.yml +++ b/backend/python/common-env/transformers/transformers.yml @@ -34,8 +34,10 @@ dependencies: - botocore==1.31.61 - certifi==2023.7.22 - charset-normalizer==3.3.0 + - compel==2.0.2 - datasets==2.14.5 - dill==0.3.7 + - diffusers==0.24.0 - einops==0.7.0 - encodec==0.1.1 - filelock==3.12.4 @@ -78,7 +80,7 @@ dependencies: - requests==2.31.0 - rouge==1.0.1 - s3transfer==0.7.0 - - safetensors==0.3.3 + - safetensors==0.4.0 - scipy==1.11.3 - six==1.16.0 - sympy==1.12 diff --git a/backend/python/diffusers/Makefile b/backend/python/diffusers/Makefile index f3f9d4e2..8e7aabba 100644 --- a/backend/python/diffusers/Makefile +++ b/backend/python/diffusers/Makefile @@ -1,8 +1,6 @@ .PHONY: diffusers diffusers: - @echo "Creating virtual environment..." - @conda env create --name diffusers --file diffusers.yml - @echo "Virtual environment created." + $(MAKE) -C ../common-env/transformers .PHONY: run run: diff --git a/backend/python/diffusers/diffusers.yml b/backend/python/diffusers/diffusers.yml deleted file mode 100644 index 48e4e62b..00000000 --- a/backend/python/diffusers/diffusers.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: diffusers -channels: - - defaults -dependencies: - - _libgcc_mutex=0.1=main - - _openmp_mutex=5.1=1_gnu - - bzip2=1.0.8=h7b6447c_0 - - ca-certificates=2023.08.22=h06a4308_0 - - ld_impl_linux-64=2.38=h1181459_1 - - libffi=3.4.4=h6a678d5_0 - - libgcc-ng=11.2.0=h1234567_1 - - libgomp=11.2.0=h1234567_1 - - libstdcxx-ng=11.2.0=h1234567_1 - - libuuid=1.41.5=h5eee18b_0 - - ncurses=6.4=h6a678d5_0 - - openssl=3.0.11=h7f8727e_2 - - pip=23.2.1=py311h06a4308_0 - - python=3.11.5=h955ad1f_0 - - readline=8.2=h5eee18b_0 - - setuptools=68.0.0=py311h06a4308_0 - - sqlite=3.41.2=h5eee18b_0 - - tk=8.6.12=h1ccaba5_0 - - tzdata=2023c=h04d1e81_0 - - wheel=0.41.2=py311h06a4308_0 - - xz=5.4.2=h5eee18b_0 - - zlib=1.2.13=h5eee18b_0 - - pip: - - accelerate>=0.11.0 - - certifi==2023.7.22 - - charset-normalizer==3.3.0 - - compel==2.0.2 - - diffusers==0.24.0 - - filelock==3.12.4 - - fsspec==2023.9.2 - - grpcio==1.59.0 - - huggingface-hub>=0.19.4 - - idna==3.4 - - importlib-metadata==6.8.0 - - jinja2==3.1.2 - - markupsafe==2.1.3 - - mpmath==1.3.0 - - networkx==3.1 - - numpy==1.26.0 - - nvidia-cublas-cu12==12.1.3.1 - - nvidia-cuda-cupti-cu12==12.1.105 - - nvidia-cuda-nvrtc-cu12==12.1.105 - - nvidia-cuda-runtime-cu12==12.1.105 - - nvidia-cudnn-cu12==8.9.2.26 - - nvidia-cufft-cu12==11.0.2.54 - - nvidia-curand-cu12==10.3.2.106 - - nvidia-cusolver-cu12==11.4.5.107 - - nvidia-cusparse-cu12==12.1.0.106 - - nvidia-nccl-cu12==2.18.1 - - nvidia-nvjitlink-cu12==12.2.140 - - nvidia-nvtx-cu12==12.1.105 - - packaging==23.2 - - pillow==10.0.1 - - protobuf==4.24.4 - - psutil==5.9.5 - - pyparsing==3.1.1 - - pyyaml==6.0.1 - - regex==2023.10.3 - - requests==2.31.0 - - safetensors==0.4.0 - - sympy==1.12 - - torch==2.1.0 - - tqdm==4.66.1 - - transformers>=4.25.1 - - triton==2.1.0 - - typing-extensions==4.8.0 - - urllib3==2.0.6 - - zipp==3.17.0 -prefix: /opt/conda/envs/diffusers diff --git a/backend/python/diffusers/run.sh b/backend/python/diffusers/run.sh index 8e3e1bbf..0ed1fb86 100755 --- a/backend/python/diffusers/run.sh +++ b/backend/python/diffusers/run.sh @@ -6,7 +6,7 @@ export PATH=$PATH:/opt/conda/bin # Activate conda environment -source activate diffusers +source activate transformers # get the directory where the bash script is located DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" diff --git a/backend/python/diffusers/test.sh b/backend/python/diffusers/test.sh index 421a1921..072fb85b 100644 --- a/backend/python/diffusers/test.sh +++ b/backend/python/diffusers/test.sh @@ -6,7 +6,7 @@ export PATH=$PATH:/opt/conda/bin # Activate conda environment -source activate diffusers +source activate transformers # get the directory where the bash script is located DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"