Did you properly install Docker and Docker nvidia toolkit ? here's the init script i'm using on my autoscaled workers:

#!/bin/sh

sudo apt-get update -y

sudo apt-get install -y \
    ca-certificates \
    curl \
    gnupg \
    lsb-release

sudo mkdir -p /etc/apt/keyrings

curl -fsSL 

 | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg

echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] 

 \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

sudo apt-get update -y

sudo apt-get install -y  docker-ce docker-ce-cli containerd.io docker-compose-plugin

sudo apt install -y nvidia-driver-515

curl -s -L 

 | \
  sudo apt-key add -

distribution=$(. /etc/os-release;echo $ID$VERSION_ID) && curl -s -L 

 | sudo apt-key add -       && curl -s -L 

 | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

sudo apt-get update -y

sudo apt-get install -y nvidia-docker2 nvidia-container-toolkit

sudo systemctl restart docker