Unanswered
Is There A Way To Control How Many Parallel Connections Are Used When Downloading From
in clearml.conf we could have:azure.storage { max_connections = 10 # containers: [ # { # account_name: "clearml" # account_key: "secret" # # container_name: # } # ] }
Then in AzureContainerConfigurations
:
` @classmethod
def from_config(cls, configuration):
...
class AzureContainerConfigurations(object):
def init(self, container_configs=None, max_connections=None):
super(AzureContainerConfigurations, self).init()
self._container_configs = container_configs or []
self.max_connections = max_connections
@classmethod
def from_config(cls, configuration):
default_account = getenv("AZURE_STORAGE_ACCOUNT")
default_key = getenv("AZURE_STORAGE_KEY")
default_container_configs = []
if default_account and default_key:
default_container_configs.append(AzureContainerConfig(
account_name=default_account, account_key=default_key
))
max_connections = configuration.get("max_connections", 10)
if configuration is None:
return cls(default_container_configs, max_connections)
containers = configuration.get("containers", list())
container_configs = [AzureContainerConfig(**entry) for entry in containers] + default_container_configs
return cls(container_configs, max_connections) `And finally:
in _AzureBlobServiceStorageDriver.download_object(...)
_ = container.blob_service.get_blob_to_path( container.name, obj.blob_name, local_path, max_connections=container.max_connections or 10, progress_callback=callback_func, )
ShakyJellyfish91 wdyt?
144 Views
0
Answers
3 years ago
one year ago