Hi AgitatedDove14 ,
I have already developed a mock test that can be somewhat similar to the pipeline we are developing. The same problem arises. Only the task is created for the first set of parameters in the for loop. Here, only the configuration text file is created for user 1. Can you reproduce it?
` from clearml import Task
from clearml.automation.controller import PipelineDecorator
@PipelineDecorator.component(
return_values=["admin_config_path"], cache=False, task_type=Task.TaskTypes.custom,
)
def admin_config_creation(config):
# import step requirements
import json
from pathlib import Path
# step code
config_filename = Path(".") / "admin_config.txt"
with config_filename.open("w") as config_file:
config_file.write(json.dumps(config, indent="\t"))
return config_filename
@PipelineDecorator.component(
return_values=["user_config_path"], cache=False, task_type=Task.TaskTypes.custom,
)
def user_config_creation(user_name, user_pass, admin_config_filename):
# import step requirements
from pathlib import Path
# step code
if user_pass.replace(" ", "").lower() != "imnotarobot":
raise ValueError("Wrong password. The configuration could not be created.")
user_config = f"(admin: {Path(admin_config_filename).stem}) Configuration created for {user_name}:\n"
user_config += (
" - ML Framework: Keras\n - Model: CNN2D\n - Problem type: Regression"
)
config_filename = Path(".") / f"{user_name}_config.txt"
with config_filename.open("w") as config_file:
config_file.write(user_config)
config_filename = str(config_filename)
print("User configuration filename casted to str?")
print(f"{config_filename} ({type(config_filename)})")
return config_filename
@PipelineDecorator.pipeline(
name="Testing pipeline's component in for loop",
project="myproject",
version="0.0.1",
)
def executing_pipeline():
# run step 1
print("Step in progress -- Admin configuration setup")
admin_config_path = admin_config_creation(
config={"name": "Guido van Rossum", "pass": "benevolent_dictator_for_life_@123"}
)
# process step 1 results
admin_config_path_1 = str(admin_config_path).replace("admin", "Gandalf")
admin_config_path_2 = str(admin_config_path).replace("admin", "Elrond")
# run step 2
users = {
"user_1": ("Im Not A Robot", admin_config_path_1),
"user_2": ("im not A robOt", admin_config_path_2),
}
users_config_filenames = {}
for name, (password, new_admin_config_path) in users.items():
print(f"Step in progress -- {name} configuration setup")
users_config_filenames[name] = user_config_creation(
user_name=name, user_pass=password, admin_config_filename=new_admin_config_path
)
print(users_config_filenames[name])
print(type(users_config_filenames[name]))
print("Users configuration files have been created. Check them out at:")
print(users_config_filenames) # I explicitly casted user configuration paths to str but they're still pathlib objects :O How?!
print("Pipeline completed!")
if name == "main":
# run the pipeline steps as subprocess on the current machine, for debugging purposes.
PipelineDecorator.debug_pipeline()
# start the pipeline execution logic.
executing_pipeline() `