api calls behaves much better
no problem to query tasks in other projects
not sure it's same use case but I will begin to ask around people
if you have any other hint/way how to query mongo and look for potential culprit - will be glad to hear
I had slightly similar scenario ~1 year and few versions back
there was some task that wrote a lot of tasks and mongo didn't took it nicely
I was able to identify to it only by questioning users and eventunaly one of them stopped to send and mongo started to come back and all return to normal
we did not come to any wise conclusion what is root cause or how to identify this
I think there are some experiments that are messing up mongodb
this logs unusual in clearml-mongo logs:
{"t":{"$date":"2023-09-19T12:15:50.685+00:00"},"s":"I", "c":"COMMAND", "id":51803, "ctx":"conn73","msg":"Slow query","attr":{"type":"command","ns":"backend.model","command":{"distinct":"model","key":"project","query":{"$and":[{"$or":[{"company":{"$in":["d1bd92a3b039400cbafc60a7a5b1e52b",null,""]}},{"company":{"$exists":false}}]},{"user":{"$in":["197aea8467d3f471fc0db98b57ed80fa"]}},{"project":{"$in":["22695d33baf64b71b5c81fd4566fc87c","7b56fe9425ed48e3864580a9b6ab1391","61fca3db40aa428cae5060ca17130bf3","058f84b972e744f3a3310790a076c102","37b307b6e98f493d8416345e9dc453d2","5162f39642754cdf84a7ad7b69a808e7","eac563bf70be4f60a8d956ee3ff18b28","0a799fbf126248e29f77a5aacfeb4f63","fba78d7926fb4fcdb3a37881de6887eb","e8c63a29db56480d8ce2f6c5123da024","93173e51f7c4438889562b8660c7d8fa","bd4c17fc84b54ccc8ac6d9537e225da8","70986016c1d9464380e99e00027b414a","024ef8164a214fc692599a5d7fa44570","0bafceb9c7e34a0caa0d83eca97e9ac1","09a64bbebfae4fcd91757c1379102782","44f1a5ec4c414f13b136e4440074d3d0","33b280dac6424bd9abe00c39e5bdd870","3489bb3f6b244123bdf31860f83a3c70","4bd25640854e453a937dc87cab3378d3","1a83e2fda41f45e99b366a989a5499d7","6474955116104effadd899e501050f51","b03c3237f1364989a5834658ee637c1c","2279539a2e0e4e0398f15e24c4e32f7b","3ab49231347847638fbbef5c172ff2ce","b0480b49ac2d471e82f730ac8d0cfaea","00acdee5aa03474384cd96bbc292fb49","dcdbb23d889d46e29451b0423ab509e7","36f7a37ee1984037aca20c5f57710f38","00865e8b66054da69b5f0a1e6ad9f4e1","64c4db6f346a47d683a797d80c789ba7","17951dd5ffe44685bbab66a56b3a06df","a603e1e6e7d44e24beafac1cc54026c5","95d2f8bf34334d34a0b628e846daa591","995a1a88614048198966a08d96bf5322","59fe86c44f2d4ec4a7ec105d536326fa","89251d2d4dfe466d979320367fdccd72","7323407da9e74cc7b8b83abce68337ad","6293a3fe8b684df99b7a81317c34d46d","cefba109f19546e096598df78e7fffe0","fa15ddbb97e74d35989e223f9a265c33","4f9b266ea49749898d85528c368aede2","4db5dc198a554317ac8fb02ef9419485","9257b8aaaae74bd5bd7b51ea7eb18d66","9f14386863a2472b80227431f0862620","6815953a07e242eba6adb963ec1c8d8b","43be253f570c49289e278da175cb5cd1","b6e78bcd33894d0f9f3e91d12358119b","aeba8850f46549598abf9f078b8a711b","35611768f55043e28dabe0fe56da4e60","62168ec681fc4e1e9936f5e3b734dbde","4b0ddb0d8d3441b48090e366c8867f73","19c2c25acf1648e9b2051fd90e7d5081","22f1d7bfffce4bffadbd159b9e1e1bc6","6c95a5edb3ea4d129ff546e7a95512b9","60303e2ff0fb4fa590bf32a42378a63d","8a84d1a065b0469db273e5af3df3ca61","138d3807150b4d98b1eef276b5cf2618","4454c8bd5feb4adea622091745ce37ea","9f493331438544e6a98e58bcbc3d80f2","625cb3c67bcb43a1b7f0090d8669a49d","ba3f3109770245409933dad13385ec5b","a0d3d4baea004c0bbed44dceceaa6049","548ce7c00d32455a8446456f20709710","1553caab5eda4a05a5ffdce7d7c83d7e","1c7015247bcf47bfa44aef8e5c8c7aa8","6a0b4c6d60404fd0b81e9667e84ce8ff","e41c17c0afa842e3b2f684b030a7e42d","6c2c8556485340f68571591a6a43ee67","f8371697002d4a39b0db2c5666a8dafc","51b73f9605a14c90890f48fb3992c794","ec56c2dab6f44ffd8cc4ed9371c6113e","46670d30b90045c9909a876bc11cc86c","f878f8c2484a40e599ed43667ac6f92d","8245e33bd67249b590d476c6533f714c","17ce0865576f489fa39a67499d045d21","f907b4d50bc94bb58af3ff83d2605620","d9aa05a73def4bc58c0570f07dac0004","bb36d88eb0e54881a42ab57577e96549","5645138aafe644c3bf6636c3787feeff","d10ede5445654565b389c6c15010bbde","a5c2f80df733464c8dcd50553f4b19ef","6a2c9c8d436147c8b065f6b9b774c567","43dd0aedde6b4e02976e8b8495100dc8","9e1e0cfb1b4b4312aedb6efcc6a511a4","207ad42d80054b04b32887541e644db3","a9bef4903a9443dd8543799333a57c59","f98170fe5612493fbb8d8a56e47439fa","76a76475a22c48a8b005ebf37e14b6a0","1a6c958cb5624cd7ad4dcb4398bb09e2","aa75b89c18224757b6e3cedc664a0104","11462c1d374a40c8b10559beac26ca3c","4238706a48b04a02b34d5eb150b6f35e","657d2117aa58434cb100d39dca50328e","6e8921d83e414a91857c40d96b76c9d6","277e8545bec54843940c342ade57cc24","578769f2102341deae6a3a305557bb61","b37d04eca8f34f56a94b971e61ff3ac6","a2b7a6f507b04c16a8aff39106e5d384","ea60ad223ece4d0babf54f3db24b6b99","76fe2c3654f0420cb215add186619259","0959d57f2d5d43159afd431dc143f6ae","4a1e741af8d04467ae4a2a1d419c61f8","a46d817f7a4c4d10a2f798f05389eb9c","7d7579251c96407794f7a10a562b9d65","6d18e6cac0cd4525a0341353e628deeb","0323301595ab4779b00fe327b7347c48","4bbaa378076140b685398d5351f6e5a6","b0aebbdf75ce49459e63c70583830fc4","6ac8f9d325b2495ea3a82dd15a70273f","85b007889d1f4a1db49be58a839d6da5","e6a920a12ad0494f9bc603a6b0d0de95","95b6bd10da154ecdac201aa25c1f35b0","3e9b78e060a84473b9894f54b2f51c91","74f6bf69433c479fabd3cf9c752f5897","e93f1e88dd964ee5a327ccc72e97416f","446c24c0c2cb49018c4597b5d2934c8a","fc28f1758aab4e6eb3091c98719252bd","4fc0e643f8654134881f4e8efb35dcee","cd15d03b0308443b9e55380e9179f35f","096b3ccfb39c4cf9b4aa780d45cf9c7c","80ae41d912254e6fbf56e57b271c9cad","abca2385f68d46d3a374d9fa1a302b18","52b77527f0ab4ed586337c430a876300","3639fe5e58bd4653834752376bfa2a5b","cb3a56dbeea140c19bd4d27abf64afaa","13325d3597184efd9ee84568fccb3fa6","a57788925f1f459a9bc3ee6b2d58e534","c1425b8eaef942e2acdb88813dbf5643","6390f95ebed240afbba47f6d085b4c5f","eed64444f5ef4715b9cf07525392c803","8809321dedb0486caeaf7a891e301896","db832fa9fafd435eb0ddea9c43e0509a","63a196f704134b0c9c0b1e020fa48666","c8b7a068ad834a8585807877f628dc57","8640f745072e43699f2dfe00f04b5eff","a2f11b010b914813a7dfddee13f6cdf9","21d80a006a4941c5b1e52573aca23944","c1531830778d4c03ac86b19f6ec78cd2","504f134871e1480cb2bae913aada2ad0","3e9ad5d6cb584c77a84e921c4d4b86ee","40a20139b950409d9df89eecb2870a55","3c8e4a1c58554b928d6622dda23a9d1a","4c3835ce01864207aa9327153972ce3a","88679a7da41c4430b7f1362edf698c3e","41e9ebd319324c5f85b326bb7a69aa61","ec7eaec3cf3e4726943476b0c4377d52","2a1e60dd1a3646789c64c44acdb7fc82","2bfd51416a3345cc97d7e6de80884de5","6f260046cae143bbafdde0e5a725ec6a","9ce2b58934114040816cfe5bf023458e","74353519a07649c8997bc67baf104b02","23f1499be79d45c48a8896a1d6a293be","470cc8b81c344957addb459f776f347c","4c2470af50e14d26b97de29c0bfc4727","5bf1f87b26484371972aa3ff0ed3154d","f5b467209eb4409da6a13615393828e0","29deebf2bb8f4bd2862cfff99bbdd36b","9880e2d8d9c94d90949c27484d54441c","e5cd45f3929a4ef288d246fbe43058d9","0c157f4092824f238ebbd6aceec81757","08e5e221ae1d40dab9fdc7e6965f07f7","bbe6304995724e49b244e6e9b29631f0","fdf085d7cd034151adfc8cb405e97489","b9b4ad5b2a45461faba1a15c121a43e8","7ef4796636294e268219532e01dc62ae","7a7e497edf054c70b1b05859bd5dbfcb","2bf4a05278b64a6a9390704c7908408d","cc8130283b1043e48df94e16a6451d46","eb5c4f05ccf442258fff5ca09aec2d63","738e388d07704ddd96aca23dda4a2469","21d015741d2a4a458887ca6704aa2eda","f86d23df38b546538b661a917e9db78c","ab16337a9a9e44ddb58516e3bdfa56a0","d18100657dda40eea09db2b3dff9a3ac","4ca15ad9999e4aa1bc7bbfcd78b392a3","2f70193bb4644b8ca9ec99e0255ffd08","1e9acc3e2951464192cd501e3021196e","f144ccd1550b4ef68ed71f02cf295111","3fb9923094cf4593a8ae296d1f03952c","9040d79e80a0442cbbb092abca039da4","70b637e0247d484fb3cba5900abefeff","06b4cf7ed2124330b739b50350080ff7","854dda4312b44e169f9513a2c950cf33","8f3667db5e86491cb528faf2a1415982","0abec2f1819a4690b435289c3e4f1c4a","9852b3f3ae7e4240a174336bfcaa9fa1","5919c93c78d449dba5fd85680ef49bba","4f0f20871805414893cc5e8e458f9d83","71c925a1f0b544f19aae961f773803d1","f4e480c7a71243b5808f20efb8b72d12","f99a6f86954d4ee99d61e308ebd630e2","f82ed3546a2e4ae8a4c0b1ed18e00be8","891ec326c91c44a09d211b937da02b35","81dccae7b2e945f6ad5bfa0ee238b5b1","30866f7add264ff0b008903c4fbeaf95","3beb1f98990a41a8b4c4308ad4cb58a9","b539bea0a35d4d69ba72e62c27c34a82","9b0cc5d299924412a63f4c8a3840f6b5","929409428f27411cb57b27a089968a49","40d6f932b48e44b7bd5d6af74c5bb8fb","f2d9bfe0700644a88dd11bfc0d83832b","49819cea4b304f4698e6d0aed8f2a651","1abb8d07dbde4326991bbd16e73a8be0","61aaf0f1ee984d8a8c008762224f88b6","458f37ee8d5c4a7d8bdfde661877212e","92d651ae4042439481de9a8088ecdca5","a35f66a7961a497984547ecac38eb50b","e9ae37dca997433a87c91e8cc3bcde2b","b5f4bda26ca44401ac350892bf654bf9","3c0bfc52123c4e299c411d93336bc186","63329825b07141eb9212d206b09bc55d","a506e759c2ce4dd99de553f2733b7d96","ce155a85c487428b8703458144d89567","0e5c3bf514e4458fbd9296189bce3ed5","da948396101b4a639b4baa4334226ac2","e9514d7c407a4fcc89288da8f2ed76e9","39b60b75c9804103b79e994cd39cc60e","78a9cdbc34324a34ae4a1444bca710ef","dd6cd6d1714d40bdb095e04bedf11c01","554dbc07601e40aa8c01e48f9a2a52ff","ae61431286f24664ac0f430b187e622c","57bbc16bb4614fc48382352df946da33","a6119b502b91496aa556c64adb2e0de6","225e2c9394614079bad4f104791d9060","bc96d2703d9743ad9b1f6cefa3a5033d","2a95577184724cbbb939b7cdc2c21030","ca882790a35749ec88ad0ac006ba462c","737e939408914352800261ff477a506a","9612f13b4a67470ea7f8a918d33fcea8","dbcff4809b144d1bb602dbb4b06fd6b7","7b40eb5788444e8d88a735bcda9b6832","760b57ec822b4302b40ba54b963e8190","2508b397cfda4bd1896d5f0c0d0a9212","6ede32bd19da4f40bd776ab205c3d4f7","60cf25a50e254745b66034746a33a014","ecef05545acc4179aa8105f947b6debc","6cb46b4d91ff43bcbe8926606aa56c9e","56c61ac520944ab39a827707cc94709e","018edd8713f148da9d18a0f43a103b3f","d36a8d50c0ab4fbe831a8f58fdab6cb3","e9c78f400f094acf8fef52ada6d5e9db","377689062bf546ad805e1497451547f8","f5adbdffaef4469ead4df12a77992674","235ad0cb41ff40e9ab416319592dadae","1879499a285443a68fb6a68a05e3fcf0","d3754e8f57624a5e85a2c8dff19505be","94da414b8d524187b8d30ea0f46ec501","2e6bf403bbc54edeb3f4e5fd65fb66be","dcc9b9def1034321bdedcb24a33fb2cb","25e6fcfecb9a4fb2b8877d40d63cee78","e71a7dc8f0104923ab6692cab75d4f70","aafd5731061f4e538fd8de52bd6acb97","657e969992a84e5fb8d349f017e1703e","1b566fd2727b42e3a9f0dac6f7ec6ae9","c010fa6b319949cc8879abd9fbff5b1e","eb79af4ef5bf4ee59471f943bd56028b","459e99586ba34a5b809c50a42ee3199b","081d0578ec404a289cd59612e6a5d55f","d65f75b40aee411b8906f0f581c100cd","5f42a55ebda342a4a548d16e76777f58","b66cd1f09b7f43c6bd4d2dff6f6228a0","643e6eea196d432ebc433b7213643040","5b9b16e70c5f41a1a3c6ff7decde6b7c","202ad7ca710646e9bb1835f80698e3e9","8aeb37057df4469783006b8a708f99f9","2f989ff11be44f2ea7b14726c85816af","75a4d07d8b944ed98e7e177681758440","7ea1add0096e4f9ca672dae095c9e27f","1739150eed824276b17a17e8ae86e0c2","99120c5d536846ffb4890bcde9ace61e","c2f7046b6e0f48b9972dd75b600b3a9a","05103e61885f465093e24f68b19c1f16","1cf33951ff1142d5bb997826c72c5809","6782939d17bf4ff1aa5d9fa8ae6e6039","564a0b79be6e49dfaf0d321379e4921b","1000fc89aa11417b9b2eafdeebe3c1d0","03bd93d811e941f2a5af1943a359065b","34cd720bd2be4179a0af18be46f92117","ed94cc258a5c43e4953354d1896618fa","584ad62bcc8842b18cdf078bb157e8d0","da893330b5f54c8d8c456d4c93fbf935","af7c030b36dd4cdbb4387fc826ff783e","3d800b144eb945c7a476110026cf6a46","66505649938244a496e532dfd83bdd08","1a44087327494af5881d6a4c1cd22e5a"]}}]}},"planSummary":"IXSCAN { project: 1 }","keysExamined":29791,"docsExamined":29562,"numYields":29,"queryHash":"4B63D9B4","planCacheKey":"18C1E7B3","reslen":26149,"locks":{"ReplicationStateTransition":{"acquireCount":{"w":30}},"Global":{"acquireCount":{"r":30}},"Database":{"acquireCount":{"r":30}},"Collection":{"acquireCount":{"r":30}},"Mutex":{"acquireCount":{"r":1}}},"storage":{},"protocol":"op_msg","durationMillis":127},"truncated":{"command":{"query":{"$and":{"2":{"project":{"$in":{"286":{"type":"string","size":37}}}}}}}},"size":{"command":38546}}
app.component.ts:138 ERROR TypeError: Cannot read properties of null (reading 'id')
at projects.effects.ts:60:56
at o.subscribe.a (switchMap.js:14:23)
at p._next (OperatorSubscriber.js:13:21)
at p.next (Subscriber.js:31:18)
at withLatestFrom.js:26:28
at p._next (OperatorSubscriber.js:13:21)
at p.next (Subscriber.js:31:18)
at filter.js:6:128
at p._next (OperatorSubscriber.js:13:21)
at p.next (Subscriber.js:31:18)
and I see also when trying to load my won task
Failed to load resource: the server responded with a status of 400 (BAD REQUEST)
Please open developer tools (F12) and see if you're getting any console errors when loading a 'stuck' experiment
I didn't saw anything useful in elasic/mongo/api
I do significany slowness to query also my experiments
no filtering for sure
if I send link to task, sometimes it loads and sometimes it's stuck
Hi @<1523701842515595264:profile|PleasantOwl46> , do you see any errors in elastic or in mongo?
Also, any chance there is filtering to only user specific tasks/projects?
What happens if you send a link to an experiment between users?