Thanks John, I read the one about the pip timeout, the problem is that I'm assume clearml runs the following command :
"pip install -r requirments.txt" and I want to know if I make clearml add the timeout flag.
I think these env varibles might be relevant to you:
CLEARML_AGENT_SKIP_PIP_VENV_INSTALL
CLEARML_AGENT_SKIP_PYTHON_ENV_INSTALL
https://clear.ml/docs/latest/docs/clearml_agent/clearml_agent_env_var
Hi CluelessFlamingo93 , I see the following traceback in your log:
`
1672308338689 bfm:0 DEBUG ━━━━━━━━━━━━━━╸ 0.7/1.8 GB 5.9 MB/s eta 0:03:16
ERROR: Exception:
Traceback (most recent call last):
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/urllib3/response.py", line 437, in _error_catcher
yield
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/urllib3/response.py", line 560, in read
data = self._fp_read(amt) if not fp_closed else b""
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/urllib3/response.py", line 526, in _fp_read
return self._fp.read(amt) if amt is not None else self._fp.read()
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/cachecontrol/filewrapper.py", line 90, in read
data = self.__fp.read(amt)
File "/home/bat/miniconda3/envs/clearml_agent/lib/python3.8/http/client.py", line 459, in read
n = self.readinto(b)
File "/home/bat/miniconda3/envs/clearml_agent/lib/python3.8/http/client.py", line 503, in readinto
n = self.fp.readinto(b)
File "/home/bat/miniconda3/envs/clearml_agent/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
File "/home/bat/miniconda3/envs/clearml_agent/lib/python3.8/ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "/home/bat/miniconda3/envs/clearml_agent/lib/python3.8/ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/cli/base_command.py", line 160, in exc_logging_wrapper
status = run_func(*args)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/cli/req_command.py", line 247, in wrapper
return func(self, options, args)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/commands/install.py", line 400, in run
requirement_set = resolver.resolve(
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 92, in resolve
result = self._result = resolver.resolve(
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/resolvelib/resolvers.py", line 481, in resolve
state = resolution.resolve(requirements, max_rounds=max_rounds)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/resolvelib/resolvers.py", line 348, in resolve
self._add_to_criteria(self.state.criteria, r, parent=None)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/resolvelib/resolvers.py", line 172, in _add_to_criteria
if not criterion.candidates:
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/resolvelib/structs.py", line 151, in bool
return bool(self._sequence)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 155, in bool
return any(self)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 143, in <genexpr>
return (c for c in iterator if id(c) not in self._incompatible_ids)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/resolution/resolvelib/found_candidates.py", line 47, in _iter_built
candidate = func()
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 206, in _make_candidate_from_link
self._link_candidate_cache[link] = LinkCandidate(
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 297, in init
super().init(
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 162, in init
self.dist = self._prepare()
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 231, in _prepare
dist = self._prepare_distribution()
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 308, in _prepare_distribution
return preparer.prepare_linked_requirement(self._ireq, parallel_builds=True)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/operations/prepare.py", line 491, in prepare_linked_requirement
return self._prepare_linked_requirement(req, parallel_builds)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/operations/prepare.py", line 536, in _prepare_linked_requirement
local_file = unpack_url(
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/operations/prepare.py", line 166, in unpack_url
file = get_http_url(
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/operations/prepare.py", line 107, in get_http_url
from_path, content_type = download(link, temp_dir.path)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/network/download.py", line 147, in call
for chunk in chunks:
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/cli/progress_bars.py", line 53, in _rich_progress_bar
for chunk in iterable:
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_internal/network/utils.py", line 63, in response_chunks
for chunk in response.raw.stream(
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/urllib3/response.py", line 621, in stream
data = self.read(amt=amt, decode_content=decode_content)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/urllib3/response.py", line 586, in read
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
File "/home/bat/miniconda3/envs/clearml_agent/lib/python3.8/contextlib.py", line 131, in exit
self.gen.throw(type, value, traceback)
File "/home/bat/.clearml/venvs-builds/3.8/lib/python3.8/site-packages/pip/_vendor/urllib3/response.py", line 442, in _error_catcher
raise ReadTimeoutError(self._pool, None, "Read timed out.")
pip._vendor.urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='download.pytorch.org', port=443): Read timed out.
ERROR: Failed installing requirements.txt: `It it possible you were having some network issues?
yes sometimes I suffer from small network issues, is there a way to make clearml have a bigger timeout when installing packages?
and if not is there a way to point it to a local package for installation or a local virtual enviroment?
I think this is the relevant one:
https://stackoverflow.com/questions/50305112/pip-install-timeout-issue
You can add it to your pip configuration so it will always be taken into account
Oh so in that case I'll need to change every agent's pip config file.
Thanks I'll look into that, but in the end we decided to add a private repo with the pytorch libraries that we need.
I've added the extra_index_url to point to our https and we changed to requirements.txt to look for that https however I'm getting this that warning I've attached and its still trying to download the packages not from my path.
how do i enable clearml-agent to look for private repos?
Regarding controlling the timeout - I think this is more of a pip configuration
CluelessFlamingo93 you can use this environment variable: https://pip.pypa.io/en/stable/topics/configuration/#environment-variables
