Unanswered
Hello! I Need To Run Clearml Pipeline With Caching Of Steps. I Specify Cache_Executed_Step=True For Each Step, But My Steps Are Not Cached And Ended With The Status - Completed.
I create a pipeline via PipelineController with adding a step as a function
pipe = PipelineController(
name=cfg.clearml.pipeline_name,
project=cfg.clearml.project_name,
target_project=True,
version=cfg.clearml.version,
add_pipeline_tags=True,
docker=cfg.clearml.dockerfile,
docker_args=DefaultMLPLATparam().docker_arg,
packages=packages,
retry_on_failure=3
)
for parameter in cfg.clearml.params:
pipe.add_parameter(
name=parameter["name"],
default=parameter["default"],
# description=parameter["description"],
)
for step in cfg.clearml.steps:
pipe.add_function_step(
name=step["name"],
function=function,
function_kwargs=step["function_kwargs"],
function_return=(
list(step["return_params"]) if step["return_params"] else None
),
project_name=cfg.clearml.project_name,
repo=cfg.clearml.repo,
repo_branch=cfg.clearml.repo_branch,
docker=step["dockerfile"],
docker_args=docker_arg,
execution_queue=task_queue_name,
cache_executed_step=True,
packages=f"{os.getcwd()}/{step['path_to_reqs']}",
parents=list(step["parents"]) if step["parents"] else None,
retry_on_failure=3
)
pipe.start(queue=None)
13 Views
0
Answers
14 days ago
13 days ago