i managed to import a custom package using the same way you did : i have added the current dir path to my system
i have a 2 steps pipeline :
- Run a function from a custom package. This function returns a Dataloader (built from torchvision.MNIST) 2) This step receives the dataloader built in the first step as a parameter ; it shows random samples from itthere has been no error to return the dataloader at the end of step1 and to import it at step2. Here is my code :
` from clearml import PipelineDecorator, Task
@PipelineDecorator.component(return_values=['dl'], cache=True,
repo='/home/xxxxxxxx/ClearML/Slack',
packages=['clearml==1.4.1'])
def step_one_IMPORT():
import sys
sys.path.insert(0,'/home/xxxxxxx/ClearML/Slack')
import omamitesh_import
print('==> STEP1: Import the custom import file')
print(f'Imported variable: {omamitesh_import.MY_GLOBAL_VAR}')
dl = omamitesh_import.my_dataloder()
print('Dataloader imported with success')
return dl
@PipelineDecorator.component(return_values=[], cache=True, parents=['step_one_IMPORT'])
def step_two_TEST_IMPORT(dl):
import numpy as np
import PIL.Image as pil
print(f'==> STEP2: Showing DL samples ({dl})')
for (i, sample) in enumerate(dl):
r = np.random.randint(32)
img = sample[0][r].view(28, 28).numpy()
img = pil.fromarray((img * 255).astype(np.uint8))
img.show()
if i > 4:
break
@PipelineDecorator.pipeline(name='220620', project='Issues Repro Pipeline', version='0.0.1',
default_queue='queue-1', pipeline_execution_queue='queue-2')
def pipeline():
# building the pipeline
dl = step_one_IMPORT()
step_two_TEST_IMPORT(dl)
if name == "main":
project_name = 'Issues Repro'
task_name = '220620'
task = Task.init(project_name=project_name, task_name=task_name)
PipelineDecorator.run_locally()
pipeline()
print('pipeline completed') `