Unanswered
Out Of A Sudden Without Any Trigger, Our Clearml Server (
I think there are some experiments that are messing up mongodb
this logs unusual in clearml-mongo logs:
{"t":{"$date":"2023-09-19T12:15:50.685+00:00"},"s":"I", "c":"COMMAND", "id":51803, "ctx":"conn73","msg":"Slow query","attr":{"type":"command","ns":"backend.model","command":{"distinct":"model","key":"project","query":{"$and":[{"$or":[{"company":{"$in":["d1bd92a3b039400cbafc60a7a5b1e52b",null,""]}},{"company":{"$exists":false}}]},{"user":{"$in":["197aea8467d3f471fc0db98b57ed80fa"]}},{"project":{"$in":["22695d33baf64b71b5c81fd4566fc87c","7b56fe9425ed48e3864580a9b6ab1391","61fca3db40aa428cae5060ca17130bf3","058f84b972e744f3a3310790a076c102","37b307b6e98f493d8416345e9dc453d2","5162f39642754cdf84a7ad7b69a808e7","eac563bf70be4f60a8d956ee3ff18b28","0a799fbf126248e29f77a5aacfeb4f63","fba78d7926fb4fcdb3a37881de6887eb","e8c63a29db56480d8ce2f6c5123da024","93173e51f7c4438889562b8660c7d8fa","bd4c17fc84b54ccc8ac6d9537e225da8","70986016c1d9464380e99e00027b414a","024ef8164a214fc692599a5d7fa44570","0bafceb9c7e34a0caa0d83eca97e9ac1","09a64bbebfae4fcd91757c1379102782","44f1a5ec4c414f13b136e4440074d3d0","33b280dac6424bd9abe00c39e5bdd870","3489bb3f6b244123bdf31860f83a3c70","4bd25640854e453a937dc87cab3378d3","1a83e2fda41f45e99b366a989a5499d7","6474955116104effadd899e501050f51","b03c3237f1364989a5834658ee637c1c","2279539a2e0e4e0398f15e24c4e32f7b","3ab49231347847638fbbef5c172ff2ce","b0480b49ac2d471e82f730ac8d0cfaea","00acdee5aa03474384cd96bbc292fb49","dcdbb23d889d46e29451b0423ab509e7","36f7a37ee1984037aca20c5f57710f38","00865e8b66054da69b5f0a1e6ad9f4e1","64c4db6f346a47d683a797d80c789ba7","17951dd5ffe44685bbab66a56b3a06df","a603e1e6e7d44e24beafac1cc54026c5","95d2f8bf34334d34a0b628e846daa591","995a1a88614048198966a08d96bf5322","59fe86c44f2d4ec4a7ec105d536326fa","89251d2d4dfe466d979320367fdccd72","7323407da9e74cc7b8b83abce68337ad","6293a3fe8b684df99b7a81317c34d46d","cefba109f19546e096598df78e7fffe0","fa15ddbb97e74d35989e223f9a265c33","4f9b266ea49749898d85528c368aede2","4db5dc198a554317ac8fb02ef9419485","9257b8aaaae74bd5bd7b51ea7eb18d66","9f14386863a2472b80227431f0862620","6815953a07e242eba6adb963ec1c8d8b","43be253f570c49289e278da175cb5cd1","b6e78bcd33894d0f9f3e91d12358119b","aeba8850f46549598abf9f078b8a711b","35611768f55043e28dabe0fe56da4e60","62168ec681fc4e1e9936f5e3b734dbde","4b0ddb0d8d3441b48090e366c8867f73","19c2c25acf1648e9b2051fd90e7d5081","22f1d7bfffce4bffadbd159b9e1e1bc6","6c95a5edb3ea4d129ff546e7a95512b9","60303e2ff0fb4fa590bf32a42378a63d","8a84d1a065b0469db273e5af3df3ca61","138d3807150b4d98b1eef276b5cf2618","4454c8bd5feb4adea622091745ce37ea","9f493331438544e6a98e58bcbc3d80f2","625cb3c67bcb43a1b7f0090d8669a49d","ba3f3109770245409933dad13385ec5b","a0d3d4baea004c0bbed44dceceaa6049","548ce7c00d32455a8446456f20709710","1553caab5eda4a05a5ffdce7d7c83d7e","1c7015247bcf47bfa44aef8e5c8c7aa8","6a0b4c6d60404fd0b81e9667e84ce8ff","e41c17c0afa842e3b2f684b030a7e42d","6c2c8556485340f68571591a6a43ee67","f8371697002d4a39b0db2c5666a8dafc","51b73f9605a14c90890f48fb3992c794","ec56c2dab6f44ffd8cc4ed9371c6113e","46670d30b90045c9909a876bc11cc86c","f878f8c2484a40e599ed43667ac6f92d","8245e33bd67249b590d476c6533f714c","17ce0865576f489fa39a67499d045d21","f907b4d50bc94bb58af3ff83d2605620","d9aa05a73def4bc58c0570f07dac0004","bb36d88eb0e54881a42ab57577e96549","5645138aafe644c3bf6636c3787feeff","d10ede5445654565b389c6c15010bbde","a5c2f80df733464c8dcd50553f4b19ef","6a2c9c8d436147c8b065f6b9b774c567","43dd0aedde6b4e02976e8b8495100dc8","9e1e0cfb1b4b4312aedb6efcc6a511a4","207ad42d80054b04b32887541e644db3","a9bef4903a9443dd8543799333a57c59","f98170fe5612493fbb8d8a56e47439fa","76a76475a22c48a8b005ebf37e14b6a0","1a6c958cb5624cd7ad4dcb4398bb09e2","aa75b89c18224757b6e3cedc664a0104","11462c1d374a40c8b10559beac26ca3c","4238706a48b04a02b34d5eb150b6f35e","657d2117aa58434cb100d39dca50328e","6e8921d83e414a91857c40d96b76c9d6","277e8545bec54843940c342ade57cc24","578769f2102341deae6a3a305557bb61","b37d04eca8f34f56a94b971e61ff3ac6","a2b7a6f507b04c16a8aff39106e5d384","ea60ad223ece4d0babf54f3db24b6b99","76fe2c3654f0420cb215add186619259","0959d57f2d5d43159afd431dc143f6ae","4a1e741af8d04467ae4a2a1d419c61f8","a46d817f7a4c4d10a2f798f05389eb9c","7d7579251c96407794f7a10a562b9d65","6d18e6cac0cd4525a0341353e628deeb","0323301595ab4779b00fe327b7347c48","4bbaa378076140b685398d5351f6e5a6","b0aebbdf75ce49459e63c70583830fc4","6ac8f9d325b2495ea3a82dd15a70273f","85b007889d1f4a1db49be58a839d6da5","e6a920a12ad0494f9bc603a6b0d0de95","95b6bd10da154ecdac201aa25c1f35b0","3e9b78e060a84473b9894f54b2f51c91","74f6bf69433c479fabd3cf9c752f5897","e93f1e88dd964ee5a327ccc72e97416f","446c24c0c2cb49018c4597b5d2934c8a","fc28f1758aab4e6eb3091c98719252bd","4fc0e643f8654134881f4e8efb35dcee","cd15d03b0308443b9e55380e9179f35f","096b3ccfb39c4cf9b4aa780d45cf9c7c","80ae41d912254e6fbf56e57b271c9cad","abca2385f68d46d3a374d9fa1a302b18","52b77527f0ab4ed586337c430a876300","3639fe5e58bd4653834752376bfa2a5b","cb3a56dbeea140c19bd4d27abf64afaa","13325d3597184efd9ee84568fccb3fa6","a57788925f1f459a9bc3ee6b2d58e534","c1425b8eaef942e2acdb88813dbf5643","6390f95ebed240afbba47f6d085b4c5f","eed64444f5ef4715b9cf07525392c803","8809321dedb0486caeaf7a891e301896","db832fa9fafd435eb0ddea9c43e0509a","63a196f704134b0c9c0b1e020fa48666","c8b7a068ad834a8585807877f628dc57","8640f745072e43699f2dfe00f04b5eff","a2f11b010b914813a7dfddee13f6cdf9","21d80a006a4941c5b1e52573aca23944","c1531830778d4c03ac86b19f6ec78cd2","504f134871e1480cb2bae913aada2ad0","3e9ad5d6cb584c77a84e921c4d4b86ee","40a20139b950409d9df89eecb2870a55","3c8e4a1c58554b928d6622dda23a9d1a","4c3835ce01864207aa9327153972ce3a","88679a7da41c4430b7f1362edf698c3e","41e9ebd319324c5f85b326bb7a69aa61","ec7eaec3cf3e4726943476b0c4377d52","2a1e60dd1a3646789c64c44acdb7fc82","2bfd51416a3345cc97d7e6de80884de5","6f260046cae143bbafdde0e5a725ec6a","9ce2b58934114040816cfe5bf023458e","74353519a07649c8997bc67baf104b02","23f1499be79d45c48a8896a1d6a293be","470cc8b81c344957addb459f776f347c","4c2470af50e14d26b97de29c0bfc4727","5bf1f87b26484371972aa3ff0ed3154d","f5b467209eb4409da6a13615393828e0","29deebf2bb8f4bd2862cfff99bbdd36b","9880e2d8d9c94d90949c27484d54441c","e5cd45f3929a4ef288d246fbe43058d9","0c157f4092824f238ebbd6aceec81757","08e5e221ae1d40dab9fdc7e6965f07f7","bbe6304995724e49b244e6e9b29631f0","fdf085d7cd034151adfc8cb405e97489","b9b4ad5b2a45461faba1a15c121a43e8","7ef4796636294e268219532e01dc62ae","7a7e497edf054c70b1b05859bd5dbfcb","2bf4a05278b64a6a9390704c7908408d","cc8130283b1043e48df94e16a6451d46","eb5c4f05ccf442258fff5ca09aec2d63","738e388d07704ddd96aca23dda4a2469","21d015741d2a4a458887ca6704aa2eda","f86d23df38b546538b661a917e9db78c","ab16337a9a9e44ddb58516e3bdfa56a0","d18100657dda40eea09db2b3dff9a3ac","4ca15ad9999e4aa1bc7bbfcd78b392a3","2f70193bb4644b8ca9ec99e0255ffd08","1e9acc3e2951464192cd501e3021196e","f144ccd1550b4ef68ed71f02cf295111","3fb9923094cf4593a8ae296d1f03952c","9040d79e80a0442cbbb092abca039da4","70b637e0247d484fb3cba5900abefeff","06b4cf7ed2124330b739b50350080ff7","854dda4312b44e169f9513a2c950cf33","8f3667db5e86491cb528faf2a1415982","0abec2f1819a4690b435289c3e4f1c4a","9852b3f3ae7e4240a174336bfcaa9fa1","5919c93c78d449dba5fd85680ef49bba","4f0f20871805414893cc5e8e458f9d83","71c925a1f0b544f19aae961f773803d1","f4e480c7a71243b5808f20efb8b72d12","f99a6f86954d4ee99d61e308ebd630e2","f82ed3546a2e4ae8a4c0b1ed18e00be8","891ec326c91c44a09d211b937da02b35","81dccae7b2e945f6ad5bfa0ee238b5b1","30866f7add264ff0b008903c4fbeaf95","3beb1f98990a41a8b4c4308ad4cb58a9","b539bea0a35d4d69ba72e62c27c34a82","9b0cc5d299924412a63f4c8a3840f6b5","929409428f27411cb57b27a089968a49","40d6f932b48e44b7bd5d6af74c5bb8fb","f2d9bfe0700644a88dd11bfc0d83832b","49819cea4b304f4698e6d0aed8f2a651","1abb8d07dbde4326991bbd16e73a8be0","61aaf0f1ee984d8a8c008762224f88b6","458f37ee8d5c4a7d8bdfde661877212e","92d651ae4042439481de9a8088ecdca5","a35f66a7961a497984547ecac38eb50b","e9ae37dca997433a87c91e8cc3bcde2b","b5f4bda26ca44401ac350892bf654bf9","3c0bfc52123c4e299c411d93336bc186","63329825b07141eb9212d206b09bc55d","a506e759c2ce4dd99de553f2733b7d96","ce155a85c487428b8703458144d89567","0e5c3bf514e4458fbd9296189bce3ed5","da948396101b4a639b4baa4334226ac2","e9514d7c407a4fcc89288da8f2ed76e9","39b60b75c9804103b79e994cd39cc60e","78a9cdbc34324a34ae4a1444bca710ef","dd6cd6d1714d40bdb095e04bedf11c01","554dbc07601e40aa8c01e48f9a2a52ff","ae61431286f24664ac0f430b187e622c","57bbc16bb4614fc48382352df946da33","a6119b502b91496aa556c64adb2e0de6","225e2c9394614079bad4f104791d9060","bc96d2703d9743ad9b1f6cefa3a5033d","2a95577184724cbbb939b7cdc2c21030","ca882790a35749ec88ad0ac006ba462c","737e939408914352800261ff477a506a","9612f13b4a67470ea7f8a918d33fcea8","dbcff4809b144d1bb602dbb4b06fd6b7","7b40eb5788444e8d88a735bcda9b6832","760b57ec822b4302b40ba54b963e8190","2508b397cfda4bd1896d5f0c0d0a9212","6ede32bd19da4f40bd776ab205c3d4f7","60cf25a50e254745b66034746a33a014","ecef05545acc4179aa8105f947b6debc","6cb46b4d91ff43bcbe8926606aa56c9e","56c61ac520944ab39a827707cc94709e","018edd8713f148da9d18a0f43a103b3f","d36a8d50c0ab4fbe831a8f58fdab6cb3","e9c78f400f094acf8fef52ada6d5e9db","377689062bf546ad805e1497451547f8","f5adbdffaef4469ead4df12a77992674","235ad0cb41ff40e9ab416319592dadae","1879499a285443a68fb6a68a05e3fcf0","d3754e8f57624a5e85a2c8dff19505be","94da414b8d524187b8d30ea0f46ec501","2e6bf403bbc54edeb3f4e5fd65fb66be","dcc9b9def1034321bdedcb24a33fb2cb","25e6fcfecb9a4fb2b8877d40d63cee78","e71a7dc8f0104923ab6692cab75d4f70","aafd5731061f4e538fd8de52bd6acb97","657e969992a84e5fb8d349f017e1703e","1b566fd2727b42e3a9f0dac6f7ec6ae9","c010fa6b319949cc8879abd9fbff5b1e","eb79af4ef5bf4ee59471f943bd56028b","459e99586ba34a5b809c50a42ee3199b","081d0578ec404a289cd59612e6a5d55f","d65f75b40aee411b8906f0f581c100cd","5f42a55ebda342a4a548d16e76777f58","b66cd1f09b7f43c6bd4d2dff6f6228a0","643e6eea196d432ebc433b7213643040","5b9b16e70c5f41a1a3c6ff7decde6b7c","202ad7ca710646e9bb1835f80698e3e9","8aeb37057df4469783006b8a708f99f9","2f989ff11be44f2ea7b14726c85816af","75a4d07d8b944ed98e7e177681758440","7ea1add0096e4f9ca672dae095c9e27f","1739150eed824276b17a17e8ae86e0c2","99120c5d536846ffb4890bcde9ace61e","c2f7046b6e0f48b9972dd75b600b3a9a","05103e61885f465093e24f68b19c1f16","1cf33951ff1142d5bb997826c72c5809","6782939d17bf4ff1aa5d9fa8ae6e6039","564a0b79be6e49dfaf0d321379e4921b","1000fc89aa11417b9b2eafdeebe3c1d0","03bd93d811e941f2a5af1943a359065b","34cd720bd2be4179a0af18be46f92117","ed94cc258a5c43e4953354d1896618fa","584ad62bcc8842b18cdf078bb157e8d0","da893330b5f54c8d8c456d4c93fbf935","af7c030b36dd4cdbb4387fc826ff783e","3d800b144eb945c7a476110026cf6a46","66505649938244a496e532dfd83bdd08","1a44087327494af5881d6a4c1cd22e5a"]}}]}},"planSummary":"IXSCAN { project: 1 }","keysExamined":29791,"docsExamined":29562,"numYields":29,"queryHash":"4B63D9B4","planCacheKey":"18C1E7B3","reslen":26149,"locks":{"ReplicationStateTransition":{"acquireCount":{"w":30}},"Global":{"acquireCount":{"r":30}},"Database":{"acquireCount":{"r":30}},"Collection":{"acquireCount":{"r":30}},"Mutex":{"acquireCount":{"r":1}}},"storage":{},"protocol":"op_msg","durationMillis":127},"truncated":{"command":{"query":{"$and":{"2":{"project":{"$in":{"286":{"type":"string","size":37}}}}}}}},"size":{"command":38546}}
140 Views
0
Answers
one year ago
one year ago