content: ARG FROM_TAG
  ARG NUM_BUILD_CORES=8
  ARG DOCKER_VER
  FROM espnet/espnet:${FROM_TAG} AS cuda_builder
  LABEL maintainer "Nelson Yalta <nyalta21@gmail.com>"
  
  ## FROM CUDA 11.7 base
  ## [https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/11.7.1/ubuntu2204/base/Dockerfile]
  ENV NVARCH x86_64
  
  ENV NVIDIA_REQUIRE_CUDA "cuda>=11.7 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471 brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471 brand=geforce,driver>=470,driver<471 brand=geforcertx,driver>=470,driver<471 brand=quadro,driver>=470,driver<471 brand=quadrortx,driver>=470,driver<471 brand=titan,driver>=470,driver<471 brand=titanrtx,driver>=470,driver<471 brand=tesla,driver>=510,driver<511 brand=unknown,driver>=510,driver<511 brand=nvidia,driver>=510,driver<511 brand=nvidiartx,driver>=510,driver<511 brand=geforce,driver>=510,driver<511 brand=geforcertx,driver>=510,driver<511 brand=quadro,driver>=510,driver<511 brand=quadrortx,driver>=510,driver<511 brand=titan,driver>=510,driver<511 brand=titanrtx,driver>=510,driver<511"
  ENV NV_CUDA_CUDART_VERSION 11.7.99-1
  ENV NV_CUDA_COMPAT_PACKAGE cuda-compat-11-7
  
  RUN apt-get update && apt-get install -y --no-install-recommends \
      gnupg2 curl ca-certificates && \
      curl -fsSLO https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/${NVARCH}/cuda-keyring_1.0-1_all.deb && \
      dpkg -i cuda-keyring_1.0-1_all.deb && \
      apt-get purge --autoremove -y curl \
      && rm -rf /var/lib/apt/lists/*
  
  ENV CUDA_VERSION 11.7.1
  
  # For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
  RUN apt-get update && apt-get install -y --no-install-recommends \
      cuda-cudart-11-7=${NV_CUDA_CUDART_VERSION} \
      ${NV_CUDA_COMPAT_PACKAGE} \
      && rm -rf /var/lib/apt/lists/*
  
  # Required for nvidia-docker v1
  RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \
      && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
  
  ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
  ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
  
  # nvidia-container-runtime
  ENV NVIDIA_VISIBLE_DEVICES all
  ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
  
  ENV CUDA_HOME /usr/local/cuda
  
  ## FROM CUDA 11.7.1 devel
  ## [https://gitlab.com/nvidia/container-images/cuda/-/blob/master/dist/11.7.1/ubuntu2204/devel/Dockerfile]
  ENV NV_LIBCUBLAS_DEV_VERSION 11.10.3.66-1
  ENV NV_CUDA_CUDART_DEV_VERSION 11.7.99-1
  ENV NV_CUDA_LIB_VERSION "11.7.1-1"
  ENV NV_NVML_DEV_VERSION 11.7.91-1
  ENV NV_NVPROF_VERSION 11.7.101-1
  ENV NV_NVPROF_DEV_PACKAGE cuda-nvprof-11-7=${NV_NVPROF_VERSION}
  ENV NV_LIBNPP_DEV_VERSION 11.7.4.75-1
  ENV NV_LIBNPP_DEV_PACKAGE libnpp-dev-11-7=${NV_LIBNPP_DEV_VERSION}
  ENV NV_LIBCUSPARSE_DEV_VERSION 11.7.4.91-1
  ENV NV_LIBCUBLAS_DEV_PACKAGE_NAME libcublas-dev-11-7
  ENV NV_LIBCUBLAS_DEV_PACKAGE ${NV_LIBCUBLAS_DEV_PACKAGE_NAME}=${NV_LIBCUBLAS_DEV_VERSION}
  ENV NV_LIBNCCL_DEV_PACKAGE_NAME libnccl-dev
  ENV NV_LIBNCCL_DEV_PACKAGE_VERSION 2.13.4-1
  ENV NCCL_VERSION 2.13.4-1
  ENV NV_LIBNCCL_DEV_PACKAGE ${NV_LIBNCCL_DEV_PACKAGE_NAME}=${NV_LIBNCCL_DEV_PACKAGE_VERSION}+cuda11.7
  ENV NV_LIBNCCL_PACKAGE_NAME libnccl2
  ENV NV_LIBNCCL_PACKAGE_VERSION 2.13.4-1
  ENV NCCL_VERSION 2.13.4-1
  ENV NV_LIBNCCL_PACKAGE ${NV_LIBNCCL_PACKAGE_NAME}=${NV_LIBNCCL_PACKAGE_VERSION}+cuda11.7
  
  
  
  RUN apt-get update && apt-get install -y --no-install-recommends \
      cuda-cudart-dev-11-7=${NV_CUDA_CUDART_DEV_VERSION} \
      cuda-command-line-tools-11-7=${NV_CUDA_LIB_VERSION} \
      cuda-minimal-build-11-7=${NV_CUDA_LIB_VERSION} \
      cuda-libraries-dev-11-7=${NV_CUDA_LIB_VERSION} \
      cuda-nvml-dev-11-7=${NV_NVML_DEV_VERSION} \
      ${NV_NVPROF_DEV_PACKAGE} \
      ${NV_LIBNPP_DEV_PACKAGE} \
      libcusparse-dev-11-7=${NV_LIBCUSPARSE_DEV_VERSION} \
      ${NV_LIBCUBLAS_DEV_PACKAGE} \
      ${NV_LIBNCCL_DEV_PACKAGE} \
      ${NV_LIBNCCL_PACKAGE} \
      && rm -rf /var/lib/apt/lists/*
  
  # Keep apt from auto upgrading the cublas and nccl packages. See https://gitlab.com/nvidia/container-images/cuda/-/issues/88
  RUN apt-mark hold ${NV_LIBCUBLAS_DEV_PACKAGE_NAME} ${NV_LIBNCCL_DEV_PACKAGE_NAME}
  ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs
  
  WORKDIR /
  