Sometimes I notice that at the end of an experiment clearml keeps hanging (something with repository detection?) and the script does not end. Do more people see this? Especially in our continuous integration pipeline this give problems because tests are getting a time out. To give a bit more detail it keeps hanging at the task.close() line.

Here the relevant part of the stack trace. I hope it is helpful
` self = <clearml.task.Task object at 0x7febb2082128>

def close(self):
    """
    Close the current Task. Enables you to manually shutdown the task.

    .. warning::
       Only call :meth:`Task.close` if you are certain the Task is not needed.
    """
    if self._at_exit_called:
        return

    # store is main before we call at_exit, because will will Null it
    is_main = self.is_main_task()
    is_sub_process = self.__is_subprocess()

    # wait for repository detection (5 minutes should be reasonable time to detect all packages)
    if self._logger and not self.__is_subprocess():
        self._wait_for_repo_detection(timeout=300.)

  self.__shutdown()

venv/lib/python3.6/site-packages/clearml/task.py:1415:

self = <clearml.task.Task object at 0x7febb2082128>

def __shutdown(self):
    """
    Will happen automatically once we exit code, i.e. atexit
    :return:
    """
    # protect sub-process at_exit
    if self._at_exit_called:
        # if we are called twice (signal in the middle of the shutdown),
        # make sure we flush stdout, this is the best we can do.
        if self._at_exit_called == get_current_thread_id() and self._logger and self.__is_subprocess():
            self._logger.set_flush_period(None)
            # noinspection PyProtectedMember
            self._logger._close_stdout_handler(wait=True)
            self._at_exit_called = True
        return

    # from here only a single thread can re-enter
    self._at_exit_called = get_current_thread_id()

    # disable lock on signal callbacks, to avoid deadlocks.
    if self.__exit_hook and self.__exit_hook.signal is not None:
        self.__edit_lock = False

    is_sub_process = self.__is_subprocess()

    # noinspection PyBroadException
    try:
        wait_for_uploads = True
        # first thing mark task as stopped, so we will not end up with "running" on lost tasks
        # if we are running remotely, the daemon will take care of it
        task_status = None
        wait_for_std_log = True
        if (not running_remotely() or DEBUG_SIMULATE_REMOTE_TASK.get()) \
                and self.is_main_task() and not is_sub_process:
            # check if we crashed, ot the signal is not interrupt (manual break)
            task_status = ('stopped', )
            if self.__exit_hook:
                is_exception = self.__exit_hook.exception
                # check if we are running inside a debugger
                if not is_exception and sys.modules.get('pydevd'):
                    # noinspection PyBroadException
                    try:
                        is_exception = sys.last_type
                    except Exception:
                        pass

                # only if we have an exception (and not ctrl-break) or signal is not SIGTERM / SIGINT
                if (is_exception and not isinstance(is_exception, KeyboardInterrupt)
                    and is_exception != KeyboardInterrupt) \
                        or (not self.__exit_hook.remote_user_aborted and
                            self.__exit_hook.signal not in (None, 2, 15)):
                    task_status = (
                        'failed',
                        'Exception {}'.format(is_exception) if is_exception else
                        'Signal {}'.format(self.__exit_hook.signal))
                    wait_for_uploads = False
                else:
                    wait_for_uploads = (self.__exit_hook.remote_user_aborted or self.__exit_hook.signal is None)
                    if not self.__exit_hook.remote_user_aborted and self.__exit_hook.signal is None and \
                            not is_exception:
                        task_status = ('completed', )
                    else:
                        task_status = ('stopped', )
                        # user aborted. do not bother flushing the stdout logs
                        wait_for_std_log = self.__exit_hook.signal is not None

        # wait for repository detection (if we didn't crash)
        if wait_for_uploads and self._logger:
            # we should print summary here
            self._summary_artifacts()
            # make sure that if we crashed the thread we are not waiting forever
            if not is_sub_process:
                self._wait_for_repo_detection(timeout=10.)

        # kill the repo thread (negative timeout, do not wait), if it hasn't finished yet.
        if not is_sub_process:
            self._wait_for_repo_detection(timeout=-1)

        # wait for uploads
        print_done_waiting = False
        if wait_for_uploads and (BackendModel.get_num_results() > 0 or
                                 (self.__reporter and self.__reporter.events_waiting())):
            self.log.info('Waiting to finish uploads')
            print_done_waiting = True
        # from here, do not send log in background thread
        if wait_for_uploads:

          self.flush(wait_for_uploads=True)

venv/lib/python3.6/site-packages/clearml/task.py:3022:

self = <clearml.task.Task object at 0x7febb2082128>, wait_for_uploads = True

def flush(self, wait_for_uploads=False):
    # type: (bool) -> bool
    """
    Flush any outstanding reports or console logs.

    :param bool wait_for_uploads: Wait for all outstanding uploads to complete

        - ``True`` - Wait
        - ``False`` - Do not wait (default)
    """

    # make sure model upload is done
    if BackendModel.get_num_results() > 0 and wait_for_uploads:
        BackendModel.wait_for_results()

    # flush any outstanding logs
    if self._logger:
        # noinspection PyProtectedMember
        self._logger._flush_stdout_handler()
    if self.__reporter:
        self.__reporter.flush()
        if wait_for_uploads:

          self.__reporter.wait_for_events()

venv/lib/python3.6/site-packages/clearml/task.py:1371:

self = <clearml.backend_interface.metrics.reporter.Reporter object at 0x7febb1e96668>
timeout = None

def wait_for_events(self, timeout=None):
    if self._report_service:

      return self._report_service.wait_for_events(timeout=timeout)

venv/lib/python3.6/site-packages/clearml/backend_interface/metrics/reporter.py:223:

self = <clearml.backend_interface.metrics.reporter.BackgroundReportService object at 0x7febb1e96630>
timeout = None

def wait_for_events(self, timeout=None):
    # noinspection PyProtectedMember
    if self._is_subprocess_mode_and_not_parent_process():
        while self._queue and not self._queue.empty():
            sleep(0.1)
        return
    self._empty_state_event.clear()

  return self._empty_state_event.wait(timeout)

venv/lib/python3.6/site-packages/clearml/backend_interface/metrics/reporter.py:80:

self = <clearml.utilities.process.mp.SafeEvent object at 0x7febb1e96128>
timeout = None

def wait(self, timeout=None):

  return self._event.wait(timeout=timeout)

venv/lib/python3.6/site-packages/clearml/utilities/process/mp.py:241:

self = <multiprocessing.synchronize.Event object at 0x7febb1e967f0>
timeout = None

def wait(self, timeout=None):
    with self._cond:
        if self._flag.acquire(False):
            self._flag.release()
        else:

          self._cond.wait(timeout)

/usr/lib/python3.6/multiprocessing/synchronize.py:360:

self = <Condition(<Lock(owner=None)>, 0)>, timeout = None

def wait(self, timeout=None):
    assert self._lock._semlock._is_mine(), \
           'must acquire() condition before using wait()'

    # indicate that this thread is going to sleep
    self._sleeping_count.release()

    # release lock
    count = self._lock._semlock._count()
    for i in range(count):
        self._lock.release()

    try:
        # wait for notification or timeout

      return self._wait_semaphore.acquire(True, timeout)

/usr/lib/python3.6/multiprocessing/synchronize.py:261:

self = <clearml.task.Task.__register_at_exit.<locals>.ExitHooks object at 0x7febb207ecc0>
sig = 2, frame = <frame object at 0x7feba4008f68>

def signal_handler(self, sig, frame):
    self.signal = sig

    org_handler = self._org_handlers.get(sig)
    signal.signal(sig, org_handler or signal.SIG_DFL)

    # if this is a sig term, we wait until __at_exit is called (basically do nothing)
    if sig == signal.SIGINT:
        # return original handler result

      return org_handler if not callable(org_handler) else org_handler(sig, frame)

E KeyboardInterrupt

venv/lib/python3.6/site-packages/clearml/task.py:3205: KeyboardInterrupt
================ 7 deselected, 8 warnings in 551.67s (0:09:11) ================= `
As you can see I pressed Ctrl+C after more than 9 minutes. Often this test is finished within 5 seconds, but sometimes not...