Thanks @<1523701070390366208:profile|CostlyOstrich36> , I've figured this out and specified how to build the ClearML agent now with the correct python version.

/usr/lib/python3/dist-packages/requests/__init__.py:89: RequestsDependencyWarning: urllib3 (1.25.8) or chardet (5.1.0) doesn't match a supported version!
  warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported "
Current configuration (clearml_agent v1.7.0, location: /tmp/.clearml_agent.r_86l8jq.cfg):
----------------------
agent.worker_id = megan-vm
agent.worker_name = vm
agent.force_git_ssh_protocol = true
agent.python_binary = 
agent.package_manager.type = poetry
agent.package_manager.pip_version.0 = <20.2 ; python_version < '3.10'
agent.package_manager.pip_version.1 = <22.3 ; python_version >\= '3.10'
agent.package_manager.system_site_packages = false
agent.package_manager.force_upgrade = false
agent.package_manager.conda_channels.0 = pytorch
agent.package_manager.conda_channels.1 = conda-forge
agent.package_manager.conda_channels.2 = defaults
agent.package_manager.priority_optional_packages.0 = pygobject
agent.package_manager.torch_nightly = false
agent.package_manager.poetry_files_from_repo_working_dir = false
agent.venvs_dir = /home/megan/.clearml/venvs-builds
agent.venvs_cache.max_entries = 10
agent.venvs_cache.free_space_threshold_gb = 2.0
agent.venvs_cache.path = ~/.clearml/venvs-cache
agent.vcs_cache.enabled = true
agent.vcs_cache.path = /home/megan/.clearml/vcs-cache
agent.venv_update.enabled = false
agent.pip_download_cache.enabled = true
agent.pip_download_cache.path = /home/megan/.clearml/pip-download-cache
agent.translate_ssh = true
agent.reload_config = false
agent.docker_pip_cache = /home/megan/.clearml/pip-cache
agent.docker_apt_cache = /home/megan/.clearml/apt-cache
agent.docker_force_pull = false
agent.default_docker.image = nvidia/cuda:11.0.3-cudnn8-runtime-ubuntu20.04
agent.enable_task_env = false
agent.hide_docker_command_env_vars.enabled = true
agent.hide_docker_command_env_vars.parse_embedded_urls = true
agent.abort_callback_max_timeout = 1800
agent.docker_internal_mounts.sdk_cache = /clearml_agent_cache
agent.docker_internal_mounts.apt_cache = /var/cache/apt/archives
agent.docker_internal_mounts.ssh_folder = ~/.ssh
agent.docker_internal_mounts.ssh_ro_folder = /.ssh
agent.docker_internal_mounts.pip_cache = /root/.cache/pip
agent.docker_internal_mounts.poetry_cache = /root/.cache/pypoetry
agent.docker_internal_mounts.vcs_cache = /root/.clearml/vcs-cache
agent.docker_internal_mounts.venv_build = ~/.clearml/venvs-builds
agent.docker_internal_mounts.pip_download = /root/.clearml/pip-download-cache
agent.apply_environment = true
agent.apply_files = true
agent.custom_build_script = 
agent.disable_task_docker_override = false
agent.git_user = 
agent.git_host = 
agent.default_python = 3.8
agent.cuda_version = 0
agent.cudnn_version = 0
api.version = 1.5
api.verify_certificate = true
api.default_version = 1.5
api.http.max_req_size = 15728640
api.http.retries.total = 240
api.http.retries.connect = 240
api.http.retries.read = 240
api.http.retries.redirect = 240
api.http.retries.status = 240
api.http.retries.backoff_factor = 1.0
api.http.retries.backoff_max = 120.0
api.http.wait_on_maintenance_forever = true
api.http.pool_maxsize = 512
api.http.pool_connections = 512
api.web_server = REMOVED
api.api_server = REMOVED
api.files_server = REMOVED
api.credentials.access_key = REMOVED
api.host = REMOVED
sdk.storage.cache.default_base_dir = ~/.clearml/cache
sdk.storage.cache.size.min_free_bytes = 10GB
sdk.storage.direct_access.0.url = file://*
sdk.metrics.file_history_size = 100
sdk.metrics.matplotlib_untitled_history_size = 100
sdk.metrics.images.format = JPEG
sdk.metrics.images.quality = 87
sdk.metrics.images.subsampling = 0
sdk.metrics.tensorboard_single_series_per_graph = false
sdk.network.metrics.file_upload_threads = 4
sdk.network.metrics.file_upload_starvation_warning_sec = 120
sdk.network.iteration.max_retries_on_server_error = 5
sdk.network.iteration.retry_backoff_factor_sec = 10
sdk.network.file_upload_retries = 3
sdk.aws.s3.key = 
sdk.aws.s3.region = 
sdk.aws.s3.use_credentials_chain = true
sdk.aws.boto3.pool_connections = 512
sdk.aws.boto3.max_multipart_concurrency = 16
sdk.log.null_log_propagate = false
sdk.log.task_log_buffer_capacity = 66
sdk.log.disable_urllib3_info = true
sdk.development.task_reuse_time_window_in_hours = 72.0
sdk.development.vcs_repo_detect_async = true
sdk.development.store_uncommitted_code_diff = true
sdk.development.support_stopping = true
sdk.development.default_output_uri = 
sdk.development.force_analyze_entire_repo = false
sdk.development.suppress_update_message = false
sdk.development.detect_with_pip_freeze = false
sdk.development.worker.report_period_sec = 2
sdk.development.worker.ping_period_sec = 30
sdk.development.worker.log_stdout = true
sdk.development.worker.report_global_mem_used = false
sdk.development.worker.report_event_flush_threshold = 100
sdk.development.worker.console_cr_flush_period = 10
sdk.apply_environment = false
sdk.apply_files = false
Executing task id [c4d9ca4361e54816b0ee4ea93d3e1dce]:
repository = REMOVED
branch = REMOVED
version_num = b75e26e30470a5087ef38caa32f285c5b394a543
tag = 
docker_cmd = 
entry_point = clearml_pipeline/schedule.py
working_dir = .
/usr/bin/python3.10: No module named virtualenv
clearml_agent: ERROR: Command '['python3.10', '-m', 'virtualenv', '/home/megan/.clearml/venvs-builds/3.10']' returned non-zero exit status 1.

This is schedule.py where the schedule_task_id is the id for a pipeline that has previously been run and now we want to schedule it:

from clearml import Task
from clearml.automation import TaskScheduler

if __name__ == "__main__":

    scheduler = TaskScheduler(force_create_task_name='megan-test-remote-pipeline')

    scheduler.add_task(
        name='Simple Pipeline Schedule Run Test',
        schedule_task_id='94a7e898bb3f4494a31828187710f5bd',
        queue='megan_testing',
        minute=2,
        weekdays=['monday', 'tuesday', 'wednesday', 'thursday', 'friday'],
        recurring=True,
        execute_immediately=True
    )

    scheduler.start_remotely(queue='megan-testing')

    print('This line will run remotely')

Hi @<1654294828365647872:profile|GorgeousShrimp11> , it appears the issue is due to running with different python versions. It looks like the python you're running the agent on, doesn't have virtualenv

I've tried changing the clearml.conf file to specify the python_binary with:

 python_binary: "usr/bin/python3"

But, still getting the same error.