I tried adding
Task.add_requirements("cudatoolkit==12.2")#replacing pip install cudatoolkit==12.2

but then got
...
agent.package_manager.torch_nightly = false
agent.package_manager.poetry_files_from_repo_working_dir = false
agent.venvs_dir = /home/rakefet/.clearml/venvs-builds
agent.venvs_cache.max_entries = 10
agent.venvs_cache.free_space_threshold_gb = 2.0
agent.venvs_cache.path = ~/.clearml/venvs-cache
agent.vcs_cache.enabled = true
agent.vcs_cache.path = /home/rakefet/.clearml/vcs-cache
agent.venv_update.enabled = false
agent.pip_download_cache.enabled = true
agent.pip_download_cache.path = /home/rakefet/.clearml/pip-download-cache
agent.translate_ssh = true
agent.reload_config = false
agent.docker_pip_cache = /home/rakefet/.clearml/pip-cache
agent.docker_apt_cache = /home/rakefet/.clearml/apt-cache
agent.docker_force_pull = false
agent.default_docker.image = nvidia/cuda:10.2-cudnn7-runtime-ubuntu18.04
agent.enable_task_env = false
agent.hide_docker_command_env_vars.enabled = true
agent.hide_docker_command_env_vars.parse_embedded_urls = true
agent.abort_callback_max_timeout = 1800
agent.docker_internal_mounts.sdk_cache = /clearml_agent_cache
agent.docker_internal_mounts.apt_cache = /var/cache/apt/archives
agent.docker_internal_mounts.ssh_folder = ~/.ssh
agent.docker_internal_mounts.ssh_ro_folder = /.ssh
agent.docker_internal_mounts.pip_cache = /root/.cache/pip
agent.docker_internal_mounts.poetry_cache = /root/.cache/pypoetry
agent.docker_internal_mounts.vcs_cache = /root/.clearml/vcs-cache
agent.docker_internal_mounts.venv_build = ~/.clearml/venvs-builds
agent.docker_internal_mounts.pip_download = /root/.clearml/pip-download-cache
agent.apply_environment = true
agent.apply_files = true
agent.custom_build_script =
agent.disable_task_docker_override = false

agent.default_python = 3.10
agent.cuda_version = 122
agent.cudnn_version = 0
Executing task id [2fcc77436db34a5192806636ef04918e]:
repository = None
branch = Rakefet
version_num = ed801b047eb6a0a269a1c0a7155db4057859b74c
tag =
docker_cmd =
entry_point = examples/emotion_conversion/speech-resynt/train.py
working_dir = .
Executing Conda: /home/rakefet/miniconda3/bin/conda env remove -p /home/rakefet/.clearml/venvs-builds/3.8 --quiet --json
Remove all packages in environment /home/rakefet/.clearml/venvs-builds/3.8:
Executing Conda: /home/rakefet/miniconda3/bin/conda create --yes --mkdir --prefix /home/rakefet/.clearml/venvs-builds/3.8 python=3.8
2023-08-22 14:52:22
clearml_agent: ERROR: Command '['/home/rakefet/miniconda3/bin/conda', 'create', '--yes', '--mkdir', '--prefix', '/home/rakefet/.clearml/venvs-builds/3.8', 'python=3.8']' returned non-zero exit status 1.
2023-08-22 14:52:23
Process failed, exit code 1