Hi all,
I'm running the ClearML Helm Chart over ArgoCD. When I apply the same chart over Docker Desktop everything works fine, but when I apply it over ArgoCD, the readiness probe of the MongoDB Pod (part of the mongodb chart that's part of the ClearML distribution chart) keeps failing, consistently, without any logs, The only logging I get, directly from the mongo pod, is this -
{"t":{"$date":"2024-12-01T15:32:05.673+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:40676","uuid":"d1649899-ff2e-4c71-9da6-935d00a55865","connectionId":173711,"connectionCount":3}}
{"t":{"$date":"2024-12-01T15:32:05.675+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn173710","msg":"client metadata","attr":{"remote":"127.0.0.1:40672","client":"conn173710","doc":{"driver":{"name":"nodejs|mongosh","version":"4.8.1"},"os":{"type":"Linux","name":"linux","architecture":"x64","version":"5.15.0-125-generic"},"platform":"Node.js v16.16.0, LE (unified)","version":"4.8.1|1.5.4","application":{"name":"mongosh 1.5.4"}}}}
{"t":{"$date":"2024-12-01T15:32:05.675+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn173711","msg":"client metadata","attr":{"remote":"127.0.0.1:40676","client":"conn173711","doc":{"driver":{"name":"nodejs|mongosh","version":"4.8.1"},"os":{"type":"Linux","name":"linux","architecture":"x64","version":"5.15.0-125-generic"},"platform":"Node.js v16.16.0, LE (unified)","version":"4.8.1|1.5.4","application":{"name":"mongosh 1.5.4"}}}}
{"t":{"$date":"2024-12-01T15:32:05.679+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:40678","uuid":"21710bde-4acf-4120-a7f7-7de03c7ba390","connectionId":173712,"connectionCount":4}}
{"t":{"$date":"2024-12-01T15:32:05.684+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn173712","msg":"client metadata","attr":{"remote":"127.0.0.1:40678","client":"conn173712","doc":{"driver":{"name":"nodejs|mongosh","version":"4.8.1"},"os":{"type":"Linux","name":"linux","architecture":"x64","version":"5.15.0-125-generic"},"platform":"Node.js v16.16.0, LE (unified)","version":"4.8.1|1.5.4","application":{"name":"mongosh 1.5.4"}}}}
{"t":{"$date":"2024-12-01T15:32:05.757+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:40692","uuid":"ffc58ad5-bf78-4293-af1c-3e8c93b4890f","connectionId":173713,"connectionCount":5}}
{"t":{"$date":"2024-12-01T15:32:05.762+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn173713","msg":"client metadata","attr":{"remote":"127.0.0.1:40692","client":"conn173713","doc":{"driver":{"name":"nodejs|mongosh","version":"4.8.1"},"os":{"type":"Linux","name":"linux","architecture":"x64","version":"5.15.0-125-generic"},"platform":"Node.js v16.16.0, LE (unified)","version":"4.8.1|1.5.4","application":{"name":"mongosh 1.5.4"}}}}
{"t":{"$date":"2024-12-01T15:32:05.778+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:40706","uuid":"d1abf83e-7425-426c-8a73-dc0afa2cd3c4","connectionId":173714,"connectionCount":6}}
{"t":{"$date":"2024-12-01T15:32:05.778+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:40714","uuid":"599656f7-748c-4a10-ad55-7473b9aae1fa","connectionId":173715,"connectionCount":7}}
{"t":{"$date":"2024-12-01T15:32:05.780+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn173714","msg":"client metadata","attr":{"remote":"127.0.0.1:40706","client":"conn173714","doc":{"driver":{"name":"nodejs|mongosh","version":"4.8.1"},"os":{"type":"Linux","name":"linux","architecture":"x64","version":"5.15.0-125-generic"},"platform":"Node.js v16.16.0, LE (unified)","version":"4.8.1|1.5.4","application":{"name":"mongosh 1.5.4"}}}}
{"t":{"$date":"2024-12-01T15:32:05.780+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn173715","msg":"client metadata","attr":{"remote":"127.0.0.1:40714","client":"conn173715","doc":{"driver":{"name":"nodejs|mongosh","version":"4.8.1"},"os":{"type":"Linux","name":"linux","architecture":"x64","version":"5.15.0-125-generic"},"platform":"Node.js v16.16.0, LE (unified)","version":"4.8.1|1.5.4","application":{"name":"mongosh 1.5.4"}}}}
{"t":{"$date":"2024-12-01T15:32:05.783+00:00"},"s":"I", "c":"NETWORK", "id":22943, "ctx":"listener","msg":"Connection accepted","attr":{"remote":"127.0.0.1:40718","uuid":"350ce00f-e4ce-47a0-a1f9-a3efae75eb84","connectionId":173716,"connectionCount":8}}
{"t":{"$date":"2024-12-01T15:32:05.787+00:00"},"s":"I", "c":"NETWORK", "id":51800, "ctx":"conn173716","msg":"client metadata","attr":{"remote":"127.0.0.1:40718","client":"conn173716","doc":{"driver":{"name":"nodejs|mongosh","version":"4.8.1"},"os":{"type":"Linux","name":"linux","architecture":"x64","version":"5.15.0-125-generic"},"platform":"Node.js v16.16.0, LE (unified)","version":"4.8.1|1.5.4","application":{"name":"mongosh 1.5.4"}}}}
{"t":{"$date":"2024-12-01T15:32:12.071+00:00"},"s":"I", "c":"-", "id":20883, "ctx":"conn173709","msg":"Interrupted operation as its client disconnected","attr":{"opId":3891042}}
{"t":{"$date":"2024-12-01T15:32:12.071+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn173710","msg":"Connection ended","attr":{"remote":"127.0.0.1:40672","uuid":"b53fdcf7-a1ee-4ff7-ad4f-176bc1fa6e52","connectionId":173710,"connectionCount":7}}
{"t":{"$date":"2024-12-01T15:32:12.072+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn173712","msg":"Connection ended","attr":{"remote":"127.0.0.1:40678","uuid":"21710bde-4acf-4120-a7f7-7de03c7ba390","connectionId":173712,"connectionCount":6}}
{"t":{"$date":"2024-12-01T15:32:12.072+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn173711","msg":"Connection ended","attr":{"remote":"127.0.0.1:40676","uuid":"d1649899-ff2e-4c71-9da6-935d00a55865","connectionId":173711,"connectionCount":5}}
{"t":{"$date":"2024-12-01T15:32:12.072+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn173709","msg":"Connection ended","attr":{"remote":"127.0.0.1:40668","uuid":"e3dcc8eb-08c1-4983-bb58-d0869940abaf","connectionId":173709,"connectionCount":4}}
{"t":{"$date":"2024-12-01T15:32:12.179+00:00"},"s":"I", "c":"-", "id":20883, "ctx":"conn173713","msg":"Interrupted operation as its client disconnected","attr":{"opId":3891044}}
{"t":{"$date":"2024-12-01T15:32:12.179+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn173714","msg":"Connection ended","attr":{"remote":"127.0.0.1:40706","uuid":"d1abf83e-7425-426c-8a73-dc0afa2cd3c4","connectionId":173714,"connectionCount":1}}
{"t":{"$date":"2024-12-01T15:32:12.179+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn173715","msg":"Connection ended","attr":{"remote":"127.0.0.1:40714","uuid":"599656f7-748c-4a10-ad55-7473b9aae1fa","connectionId":173715,"connectionCount":2}}
{"t":{"$date":"2024-12-01T15:32:12.179+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn173716","msg":"Connection ended","attr":{"remote":"127.0.0.1:40718","uuid":"350ce00f-e4ce-47a0-a1f9-a3efae75eb84","connectionId":173716,"connectionCount":3}}
{"t":{"$date":"2024-12-01T15:32:12.180+00:00"},"s":"I", "c":"NETWORK", "id":22944, "ctx":"conn173713","msg":"Connection ended","attr":{"remote":"127.0.0.
(and it's looping like this infinitely).
I've tried applying different versions, I even tried to add through the Argo Application manifest a new initialDelaySeconds for the probe, hoping this might be a slow startup issue as often documented in other answers here, but to no avail (30 seconds, which seem more than reasonable, wasn't enough). I'm completely lost on this. Anyone every encountered this / something similar? thanks!