Unanswered
Hi Guys,
I Have Data ( For Folder Tree Image Classification) And We Stored That On S3 Bucket Minio
Can I Register Dataset To Clearml Without Downloading (Using Storage Manager) Data On S3 Bucket To My Local Then Upload To Clearml?
# downloading data from s3 manager = StorageManager() target_folder = manager.download_folder( local_folder='/tmp', remote_url=f'
` '
)
# upload to clearml
dataset = Dataset.create(
dataset_project=metadata[2],
dataset_name=metadata[3],
dataset_tags=tags,
output_uri=" ` ` "
)
fp_target_folder = os.path.join(target_folder, minio_s3_url)
print('>>> target_folder:', fp_target_folder)
print('>>> target_folder:', os.listdir(fp_target_folder))
create_histogram_classifier(fp_target_folder, metadata[4], dataset)
create_3d_scatter(fp_target_folder, dataset)
dataset.get_logger().report_table(series='Table', title='Table Information',table_plot=df_table)
dataset.add_files(path=fp_target_folder)
dataset.upload(show_progress=True, chunk_size=100, verbose=True)
is_success = dataset.finalize() `
105 Views
0
Answers
one year ago
one year ago