From e52c66c76eb37c67bf3ae2d30060d60eb3e46722 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 14 May 2025 19:28:30 +0200 Subject: [PATCH] chore(docs/install.sh): image changes (#5354) chore(docs): image changes Signed-off-by: Ettore Di Giacinto --- README.md | 82 ++++++++++++++++--- docs/content/docs/advanced/installer.md | 11 ++- .../docs/getting-started/container-images.md | 42 ++++------ docs/static/install.sh | 69 +++++++++------- 4 files changed, 138 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index ddeb5d65..d85f7d67 100644 --- a/README.md +++ b/README.md @@ -113,22 +113,84 @@ For more installation options, see [Installer Options](https://localai.io/docs/a Or run with docker: ### CPU only image: -```bash -docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-cpu -``` -### Nvidia GPU: -```bash -docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-12 -``` -### CPU and GPU image (bigger size): + ```bash docker run -ti --name local-ai -p 8080:8080 localai/localai:latest ``` -### AIO images (it will pre-download a set of models ready for use, see https://localai.io/basics/container/) + +### NVIDIA GPU Images: + ```bash -docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-aio-cpu +# CUDA 12.0 with core features +docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-12 + +# CUDA 12.0 with extra Python dependencies +docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-12-extras + +# CUDA 11.7 with core features +docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-11 + +# CUDA 11.7 with extra Python dependencies +docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-gpu-nvidia-cuda-11-extras + +# NVIDIA Jetson (L4T) ARM64 +docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-nvidia-l4t-arm64 ``` +### AMD GPU Images (ROCm): + +```bash +# ROCm with core features +docker run -ti --name local-ai -p 8080:8080 --device=/dev/kfd --device=/dev/dri --group-add=video localai/localai:latest-gpu-hipblas + +# ROCm with extra Python dependencies +docker run -ti --name local-ai -p 8080:8080 --device=/dev/kfd --device=/dev/dri --group-add=video localai/localai:latest-gpu-hipblas-extras +``` + +### Intel GPU Images (oneAPI): + +```bash +# Intel GPU with FP16 support +docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-gpu-intel-f16 + +# Intel GPU with FP16 support and extra dependencies +docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-gpu-intel-f16-extras + +# Intel GPU with FP32 support +docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-gpu-intel-f32 + +# Intel GPU with FP32 support and extra dependencies +docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-gpu-intel-f32-extras +``` + +### Vulkan GPU Images: + +```bash +# Vulkan with core features +docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-gpu-vulkan +``` + +### AIO Images (pre-downloaded models): + +```bash +# CPU version +docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-aio-cpu + +# NVIDIA CUDA 12 version +docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-aio-gpu-nvidia-cuda-12 + +# NVIDIA CUDA 11 version +docker run -ti --name local-ai -p 8080:8080 --gpus all localai/localai:latest-aio-gpu-nvidia-cuda-11 + +# Intel GPU version +docker run -ti --name local-ai -p 8080:8080 localai/localai:latest-aio-gpu-intel-f16 + +# AMD GPU version +docker run -ti --name local-ai -p 8080:8080 --device=/dev/kfd --device=/dev/dri --group-add=video localai/localai:latest-aio-gpu-hipblas +``` + +For more information about the AIO images and pre-downloaded models, see [Container Documentation](https://localai.io/basics/container/). + To load models: ```bash diff --git a/docs/content/docs/advanced/installer.md b/docs/content/docs/advanced/installer.md index f584da27..e473c090 100644 --- a/docs/content/docs/advanced/installer.md +++ b/docs/content/docs/advanced/installer.md @@ -23,8 +23,9 @@ List of the Environment Variables: |----------------------|--------------------------------------------------------------| | **DOCKER_INSTALL** | Set to "true" to enable the installation of Docker images. | | **USE_AIO** | Set to "true" to use the all-in-one LocalAI Docker image. | +| **USE_EXTRAS** | Set to "true" to use images with extra Python dependencies. | +| **USE_VULKAN** | Set to "true" to use Vulkan GPU support. | | **API_KEY** | Specify an API key for accessing LocalAI, if required. | -| **CORE_IMAGES** | Set to "true" to download core LocalAI images. | | **PORT** | Specifies the port on which LocalAI will run (default is 8080). | | **THREADS** | Number of processor threads the application should use. Defaults to the number of logical cores minus one. | | **VERSION** | Specifies the version of LocalAI to install. Defaults to the latest available version. | @@ -34,6 +35,14 @@ List of the Environment Variables: | **FEDERATED** | Set to "true" to share the instance with the federation (p2p token is required see [documentation]({{%relref "docs/features/distributed_inferencing" %}})) | | **FEDERATED_SERVER** | Set to "true" to run the instance as a federation server which forwards requests to the federation (p2p token is required see [documentation]({{%relref "docs/features/distributed_inferencing" %}})) | +## Image Selection + +The installer will automatically detect your GPU and select the appropriate image. By default, it uses the standard images without extra Python dependencies. You can customize the image selection using the following environment variables: + +- `USE_EXTRAS=true`: Use images with extra Python dependencies (larger images, ~17GB) +- `USE_AIO=true`: Use all-in-one images that include all dependencies +- `USE_VULKAN=true`: Use Vulkan GPU support instead of vendor-specific GPU support + ## Uninstallation To uninstall, run: diff --git a/docs/content/docs/getting-started/container-images.md b/docs/content/docs/getting-started/container-images.md index d1930805..6f4b2fc2 100644 --- a/docs/content/docs/getting-started/container-images.md +++ b/docs/content/docs/getting-started/container-images.md @@ -150,7 +150,7 @@ The AIO Images are inheriting the same environment variables as the base images Standard container images do not have pre-installed models. -Images are available with and without python dependencies. Note that images with python dependencies are bigger (in order of 17GB). +Images are available with and without python dependencies (images with the `extras` suffix). Note that images with python dependencies are bigger (in order of 17GB). Images with `core` in the tag are smaller and do not contain any python dependencies. @@ -160,10 +160,8 @@ Images with `core` in the tag are smaller and do not contain any python dependen | Description | Quay | Docker Hub | | --- | --- |-----------------------------------------------| | Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master` | `localai/localai:master` | -| Latest tag | `quay.io/go-skynet/local-ai:latest-cpu` | `localai/localai:latest-cpu` | +| Latest tag | `quay.io/go-skynet/local-ai:latest` | `localai/localai:latest` | | Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}` | `localai/localai:{{< version >}}` | -| Versioned image including FFMpeg| `quay.io/go-skynet/local-ai:{{< version >}}-ffmpeg` | `localai/localai:{{< version >}}-ffmpeg` | -| Versioned image including FFMpeg, no python | `quay.io/go-skynet/local-ai:{{< version >}}-ffmpeg-core` | `localai/localai:{{< version >}}-ffmpeg-core` | {{% /tab %}} @@ -172,10 +170,9 @@ Images with `core` in the tag are smaller and do not contain any python dependen | Description | Quay | Docker Hub | | --- | --- |-------------------------------------------------------------| | Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-cublas-cuda11` | `localai/localai:master-cublas-cuda11` | -| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-nvidia-cuda-11` | `localai/localai:latest-gpu-nvidia-cuda-11` | +| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-nvidia-cuda-11` | `localai/localai:latest-gpu-nvidia-cuda-11` | +| Latest tag with extras | `quay.io/go-skynet/local-ai:latest-gpu-nvidia-cuda-11-extras` | `localai/localai:latest-gpu-nvidia-cuda-11-extras` | | Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-cublas-cuda11` | `localai/localai:{{< version >}}-cublas-cuda11` | -| Versioned image including FFMpeg| `quay.io/go-skynet/local-ai:{{< version >}}-cublas-cuda11-ffmpeg` | `localai/localai:{{< version >}}-cublas-cuda11-ffmpeg` | -| Versioned image including FFMpeg, no python | `quay.io/go-skynet/local-ai:{{< version >}}-cublas-cuda11-ffmpeg-core` | `localai/localai:{{< version >}}-cublas-cuda11-ffmpeg-core` | {{% /tab %}} @@ -185,9 +182,8 @@ Images with `core` in the tag are smaller and do not contain any python dependen | --- | --- |-------------------------------------------------------------| | Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-cublas-cuda12` | `localai/localai:master-cublas-cuda12` | | Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-nvidia-cuda-12` | `localai/localai:latest-gpu-nvidia-cuda-12` | +| Latest tag with extras | `quay.io/go-skynet/local-ai:latest-gpu-nvidia-cuda-12-extras` | `localai/localai:latest-gpu-nvidia-cuda-12-extras` | | Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-cublas-cuda12` | `localai/localai:{{< version >}}-cublas-cuda12` | -| Versioned image including FFMpeg| `quay.io/go-skynet/local-ai:{{< version >}}-cublas-cuda12-ffmpeg` | `localai/localai:{{< version >}}-cublas-cuda12-ffmpeg` | -| Versioned image including FFMpeg, no python | `quay.io/go-skynet/local-ai:{{< version >}}-cublas-cuda12-ffmpeg-core` | `localai/localai:{{< version >}}-cublas-cuda12-ffmpeg-core` | {{% /tab %}} @@ -197,9 +193,8 @@ Images with `core` in the tag are smaller and do not contain any python dependen | --- | --- |-------------------------------------------------------------| | Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-sycl-f16` | `localai/localai:master-sycl-f16` | | Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-intel-f16` | `localai/localai:latest-gpu-intel-f16` | -| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-sycl-f16-core` | `localai/localai:{{< version >}}-sycl-f16-core` | -| Versioned image including FFMpeg| `quay.io/go-skynet/local-ai:{{< version >}}-sycl-f16-ffmpeg` | `localai/localai:{{< version >}}-sycl-f16-ffmpeg` | -| Versioned image including FFMpeg, no python | `quay.io/go-skynet/local-ai:{{< version >}}-sycl-f16-ffmpeg-core` | `localai/localai:{{< version >}}-sycl-f16-ffmpeg-core` | +| Latest tag with extras | `quay.io/go-skynet/local-ai:latest-gpu-intel-f16-extras` | `localai/localai:latest-gpu-intel-f16-extras` | +| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-sycl-f16` | `localai/localai:{{< version >}}-sycl-f16` | {{% /tab %}} @@ -209,9 +204,8 @@ Images with `core` in the tag are smaller and do not contain any python dependen | --- | --- |-------------------------------------------------------------| | Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-sycl-f32` | `localai/localai:master-sycl-f32` | | Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-intel-f32` | `localai/localai:latest-gpu-intel-f32` | -| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-sycl-f32-core` | `localai/localai:{{< version >}}-sycl-f32-core` | -| Versioned image including FFMpeg| `quay.io/go-skynet/local-ai:{{< version >}}-sycl-f32-ffmpeg` | `localai/localai:{{< version >}}-sycl-f32-ffmpeg` | -| Versioned image including FFMpeg, no python | `quay.io/go-skynet/local-ai:{{< version >}}-sycl-f32-ffmpeg-core` | `localai/localai:{{< version >}}-sycl-f32-ffmpeg-core` | +| Latest tag with extras | `quay.io/go-skynet/local-ai:latest-gpu-intel-f32-extras` | `localai/localai:latest-gpu-intel-f32-extras` | +| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-sycl-f32` | `localai/localai:{{< version >}}-sycl-f32` | {{% /tab %}} @@ -220,20 +214,18 @@ Images with `core` in the tag are smaller and do not contain any python dependen | Description | Quay | Docker Hub | | --- | --- |-------------------------------------------------------------| | Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-hipblas` | `localai/localai:master-hipblas` | -| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-hipblas` | `localai/localai:latest-gpu-hipblas` | +| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-hipblas` | `localai/localai:latest-gpu-hipblas` | +| Latest tag with extras | `quay.io/go-skynet/local-ai:latest-gpu-hipblas-extras` | `localai/localai:latest-gpu-hipblas-extras` | | Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-hipblas` | `localai/localai:{{< version >}}-hipblas` | -| Versioned image including FFMpeg| `quay.io/go-skynet/local-ai:{{< version >}}-hipblas-ffmpeg` | `localai/localai:{{< version >}}-hipblas-ffmpeg` | -| Versioned image including FFMpeg, no python | `quay.io/go-skynet/local-ai:{{< version >}}-hipblas-ffmpeg-core` | `localai/localai:{{< version >}}-hipblas-ffmpeg-core` | {{% /tab %}} - {{% tab tabName="Vulkan Images" %}} | Description | Quay | Docker Hub | | --- | --- |-------------------------------------------------------------| -| Latest images from the branch (development) | `quay.io/go-skynet/local-ai: master-vulkan-ffmpeg-core ` | `localai/localai: master-vulkan-ffmpeg-core ` | -| Latest tag | `quay.io/go-skynet/local-ai: latest-vulkan-ffmpeg-core ` | `localai/localai: latest-vulkan-ffmpeg-core` | -| Versioned image including FFMpeg, no python | `quay.io/go-skynet/local-ai:{{< version >}}-vulkan-fmpeg-core` | `localai/localai:{{< version >}}-vulkan-fmpeg-core` | +| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-vulkan` | `localai/localai:master-vulkan` | +| Latest tag | `quay.io/go-skynet/local-ai:latest-gpu-vulkan` | `localai/localai:latest-gpu-vulkan` | +| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-vulkan` | `localai/localai:{{< version >}}-vulkan` | {{% /tab %}} {{% tab tabName="Nvidia Linux for tegra" %}} @@ -242,9 +234,9 @@ These images are compatible with Nvidia ARM64 devices, such as the Jetson Nano, | Description | Quay | Docker Hub | | --- | --- |-------------------------------------------------------------| -| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64-core` | `localai/localai:master-nvidia-l4t-arm64-core` | -| Latest tag | `quay.io/go-skynet/local-ai:latest-nvidia-l4t-arm64-core` | `localai/localai:latest-nvidia-l4t-arm64-core` | -| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-nvidia-l4t-arm64-core` | `localai/localai:{{< version >}}-nvidia-l4t-arm64-core` | +| Latest images from the branch (development) | `quay.io/go-skynet/local-ai:master-nvidia-l4t-arm64` | `localai/localai:master-nvidia-l4t-arm64` | +| Latest tag | `quay.io/go-skynet/local-ai:latest-nvidia-l4t-arm64` | `localai/localai:latest-nvidia-l4t-arm64` | +| Versioned image | `quay.io/go-skynet/local-ai:{{< version >}}-nvidia-l4t-arm64` | `localai/localai:{{< version >}}-nvidia-l4t-arm64` | {{% /tab %}} diff --git a/docs/static/install.sh b/docs/static/install.sh index c39c6ba3..33e13375 100755 --- a/docs/static/install.sh +++ b/docs/static/install.sh @@ -16,6 +16,8 @@ # Environment Variables: # DOCKER_INSTALL - Set to "true" to install Docker images (default: auto-detected) # USE_AIO - Set to "true" to use the all-in-one LocalAI image (default: false) +# USE_EXTRAS - Set to "true" to use images with extra Python dependencies (default: false) +# USE_VULKAN - Set to "true" to use Vulkan GPU support (default: false) # API_KEY - API key for securing LocalAI access (default: none) # PORT - Port to run LocalAI on (default: 8080) # THREADS - Number of CPU threads to use (default: auto-detected) @@ -158,6 +160,8 @@ uninstall_localai() { # DOCKER_INSTALL - set to "true" to install Docker images # USE_AIO - set to "true" to install the all-in-one LocalAI image +# USE_EXTRAS - set to "true" to use images with extra Python dependencies +# USE_VULKAN - set to "true" to use Vulkan GPU support PORT=${PORT:-8080} docker_found=false @@ -171,6 +175,8 @@ fi DOCKER_INSTALL=${DOCKER_INSTALL:-$docker_found} USE_AIO=${USE_AIO:-false} +USE_EXTRAS=${USE_EXTRAS:-false} +USE_VULKAN=${USE_VULKAN:-false} API_KEY=${API_KEY:-} CORE_IMAGES=${CORE_IMAGES:-false} P2P_TOKEN=${P2P_TOKEN:-} @@ -404,9 +410,9 @@ install_container_toolkit() { info "Restarting Docker Daemon" $SUDO systemctl restart docker - # The NVML error arises because SELinux blocked the container’s attempts to open the GPU devices or related libraries. - # Without relaxing SELinux for the container, GPU commands like nvidia-smi report “Insufficient Permissions” - # This has been noted in NVIDIA’s documentation: + # The NVML error arises because SELinux blocked the container's attempts to open the GPU devices or related libraries. + # Without relaxing SELinux for the container, GPU commands like nvidia-smi report "Insufficient Permissions" + # This has been noted in NVIDIA's documentation: # ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/1.13.5/install-guide.html#id2 # ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/troubleshooting.html#nvml-insufficient-permissions-and-selinux case $OS_NAME in @@ -645,15 +651,6 @@ install_docker() { if $SUDO docker ps -a --format '{{.Names}}' | grep -q local-ai; then info "LocalAI Docker container already exists, replacing it..." $SUDO docker rm -f local-ai - # # Check if it is running - # if $SUDO docker ps --format '{{.Names}}' | grep -q local-ai; then - # info "LocalAI Docker container is already running." - # exit 0 - # fi - - # info "Starting LocalAI Docker container..." - # $SUDO docker start local-ai - # exit 0 fi envs="" @@ -665,11 +662,23 @@ install_docker() { fi IMAGE_TAG= - if [ "$HAS_CUDA" ]; then - IMAGE_TAG=${LOCALAI_VERSION}-cublas-cuda12-ffmpeg - # CORE - if [ "$CORE_IMAGES" = true ]; then - IMAGE_TAG=${LOCALAI_VERSION}-cublas-cuda12-ffmpeg-core + if [ "$USE_VULKAN" = true ]; then + IMAGE_TAG=${LOCALAI_VERSION}-gpu-vulkan + + info "Starting LocalAI Docker container..." + $SUDO docker run -v local-ai-data:/build/models \ + --device /dev/dri \ + --restart=always \ + -e API_KEY=$API_KEY \ + -e THREADS=$THREADS \ + $envs \ + -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG $STARTCOMMAND + elif [ "$HAS_CUDA" ]; then + # Default to CUDA 12 + IMAGE_TAG=${LOCALAI_VERSION}-cublas-cuda12 + # EXTRAS + if [ "$USE_EXTRAS" = true ]; then + IMAGE_TAG=${LOCALAI_VERSION}-cublas-cuda12-extras fi # AIO if [ "$USE_AIO" = true ]; then @@ -696,10 +705,10 @@ install_docker() { $envs \ -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG $STARTCOMMAND elif [ "$HAS_AMD" ]; then - IMAGE_TAG=${LOCALAI_VERSION}-hipblas-ffmpeg - # CORE - if [ "$CORE_IMAGES" = true ]; then - IMAGE_TAG=${LOCALAI_VERSION}-hipblas-ffmpeg-core + IMAGE_TAG=${LOCALAI_VERSION}-hipblas + # EXTRAS + if [ "$USE_EXTRAS" = true ]; then + IMAGE_TAG=${LOCALAI_VERSION}-hipblas-extras fi # AIO if [ "$USE_AIO" = true ]; then @@ -710,16 +719,18 @@ install_docker() { $SUDO docker run -v local-ai-data:/build/models \ --device /dev/dri \ --device /dev/kfd \ + --group-add=video \ --restart=always \ -e API_KEY=$API_KEY \ -e THREADS=$THREADS \ $envs \ -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG $STARTCOMMAND elif [ "$HAS_INTEL" ]; then - IMAGE_TAG=${LOCALAI_VERSION}-sycl-f32-ffmpeg - # CORE - if [ "$CORE_IMAGES" = true ]; then - IMAGE_TAG=${LOCALAI_VERSION}-sycl-f32-ffmpeg-core + # Default to FP32 for better compatibility + IMAGE_TAG=${LOCALAI_VERSION}-sycl-f32 + # EXTRAS + if [ "$USE_EXTRAS" = true ]; then + IMAGE_TAG=${LOCALAI_VERSION}-sycl-f32-extras fi # AIO if [ "$USE_AIO" = true ]; then @@ -734,12 +745,10 @@ install_docker() { -e THREADS=$THREADS \ $envs \ -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG $STARTCOMMAND + else - IMAGE_TAG=${LOCALAI_VERSION}-ffmpeg - # CORE - if [ "$CORE_IMAGES" = true ]; then - IMAGE_TAG=${LOCALAI_VERSION}-ffmpeg-core - fi + IMAGE_TAG=${LOCALAI_VERSION} + # AIO if [ "$USE_AIO" = true ]; then IMAGE_TAG=${LOCALAI_VERSION}-aio-cpu