{ "environment": { "mongodVersion": "7.0.40", "members": 3, "electionTimeoutMillis": 10000, "heartbeatIntervalMillis": 2000, "writeConcernMajorityJournalDefault": true, "defaultWriteConcern": "{ \"w\" : \"majority\", \"wtimeout\" : 0 }", "driver": "3.10.0.0", "driverInformationalVersion": "3.10.0+3e1d12614abfc181a14e388aff8daf31f88124b5", "clientSettings": { "maxConnectionPoolSize": 100, "serverSelectionTimeoutMs": 30000, "connectTimeoutMs": 30000, "heartbeatIntervalMs": 10000, "retryReads": true, "retryWrites": true, "directConnection": false }, "deadlineMs": 5000, "toleranceMs": 100, "benchProcessors": 48, "quick": false }, "latency": [ { "kind": "majority-writes-alone", "writers": 4, "result": { "acknowledged": 16488, "errors": {}, "perSecond": 1099.2, "p50": 3.573099999997794, "p95": 4.990200000000186, "p99": 6.519300000003568, "max": 47.63890000000174 } }, { "kind": "reads", "repeat": 1, "reader": "linearizable", "clients": 1, "perClientConcurrency": 1, "concurrency": 1, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 10210, "Errors": 0, "ThroughputPerSecond": 340.3333333333333, "P50": 2.9541000000026543, "P95": 3.896600000000035, "P99": 4.1414999999979045, "P999": 7.289899999996123, "Max": 9.17670000000362 }, "backgroundMajorityWrites": { "acknowledged": 1306, "errors": {}, "perSecond": 37.31428571428572, "p50": 3.5430000000051223, "p95": 4.549100000003818, "p99": 5.12949999999546, "max": 8.520200000006298 } }, { "kind": "reads", "repeat": 1, "reader": "majority", "clients": 1, "perClientConcurrency": 1, "concurrency": 1, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 58699, "Errors": 0, "ThroughputPerSecond": 1956.6333333333334, "P50": 0.5023999999975786, "P95": 0.6821000000054482, "P99": 0.804599999988568, "P999": 5.0218000000022585, "Max": 22.829299999997602 }, "backgroundMajorityWrites": { "acknowledged": 1294, "errors": {}, "perSecond": 36.97142857142857, "p50": 4.1260999999940395, "p95": 4.75170000000071, "p99": 5.0592000000033295, "max": 25.04769999999553 } }, { "kind": "reads", "repeat": 1, "reader": "linearizable", "clients": 3, "perClientConcurrency": 32, "concurrency": 96, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 368274, "Errors": 0, "ThroughputPerSecond": 12275.8, "P50": 5.789999999993597, "P95": 21.822100000004866, "P99": 26.027000000001863, "P999": 30.164399999994203, "Max": 37.61259999999311 }, "backgroundMajorityWrites": { "acknowledged": 1170, "errors": {}, "perSecond": 33.42857142857143, "p50": 5.9024999999965075, "p95": 20.604200000001583, "p99": 24.073499999998603, "max": 35.585900000005495 } }, { "kind": "reads", "repeat": 1, "reader": "majority", "clients": 3, "perClientConcurrency": 32, "concurrency": 96, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 548403, "Errors": 0, "ThroughputPerSecond": 18280.1, "P50": 1.6499999999941792, "P95": 23.03620000000228, "P99": 26.898400000005495, "P999": 39.68459999997867, "Max": 75.14900000000489 }, "backgroundMajorityWrites": { "acknowledged": 993, "errors": {}, "perSecond": 28.37142857142857, "p50": 9.104700000025332, "p95": 30.444599999987986, "p99": 37.9947000000102, "max": 52.999199999991106 } }, { "kind": "reads", "repeat": 1, "reader": "linearizable", "clients": 10, "perClientConcurrency": 32, "concurrency": 320, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 292835, "Errors": 0, "ThroughputPerSecond": 9761.166666666666, "P50": 34.906199999997625, "P95": 44.884799999999814, "P99": 49.39389999999548, "P999": 64.95539999997709, "Max": 80.83309999998892 }, "backgroundMajorityWrites": { "acknowledged": 818, "errors": {}, "perSecond": 23.37142857142857, "p50": 23.08670000001439, "p95": 45.13829999999143, "p99": 50.88939999998547, "max": 60.80760000000009 } }, { "kind": "reads", "repeat": 1, "reader": "majority", "clients": 10, "perClientConcurrency": 32, "concurrency": 320, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 419399, "Errors": 0, "ThroughputPerSecond": 13979.966666666667, "P50": 25.240099999995437, "P95": 36.05860000001849, "P99": 55.646100000012666, "P999": 74.85289999999804, "Max": 128.0853000000061 }, "backgroundMajorityWrites": { "acknowledged": 795, "errors": {}, "perSecond": 22.714285714285715, "p50": 29.864100000006147, "p95": 56.04879999998957, "p99": 60.11139999999432, "max": 70.61009999999078 } }, { "kind": "reads", "repeat": 2, "reader": "linearizable", "clients": 1, "perClientConcurrency": 1, "concurrency": 1, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 10724, "Errors": 0, "ThroughputPerSecond": 357.46666666666664, "P50": 2.7628000000258908, "P95": 3.8193000000028405, "P99": 4.101800000004005, "P999": 7.764599999994971, "Max": 19.438300000008894 }, "backgroundMajorityWrites": { "acknowledged": 1306, "errors": {}, "perSecond": 37.31428571428572, "p50": 3.4421999999904074, "p95": 4.517600000021048, "p99": 5.169999999983702, "max": 20.382500000006985 } }, { "kind": "reads", "repeat": 2, "reader": "majority", "clients": 1, "perClientConcurrency": 1, "concurrency": 1, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 57968, "Errors": 0, "ThroughputPerSecond": 1932.2666666666667, "P50": 0.5077999999630265, "P95": 0.6843999999691732, "P99": 0.7943999999552034, "P999": 5.178300000028685, "Max": 20.30999999999767 }, "backgroundMajorityWrites": { "acknowledged": 1292, "errors": {}, "perSecond": 36.91428571428571, "p50": 4.135099999955855, "p95": 4.81260000000475, "p99": 7.775599999993574, "max": 25.90649999998277 } }, { "kind": "reads", "repeat": 2, "reader": "linearizable", "clients": 3, "perClientConcurrency": 32, "concurrency": 96, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 366282, "Errors": 0, "ThroughputPerSecond": 12209.4, "P50": 5.859800000034738, "P95": 21.250700000033248, "P99": 25.462200000009034, "P999": 29.07750000001397, "Max": 41.91019999998389 }, "backgroundMajorityWrites": { "acknowledged": 1168, "errors": {}, "perSecond": 33.371428571428574, "p50": 5.832899999979418, "p95": 21.0914000000339, "p99": 24.803499999979977, "max": 29.9033000000054 } }, { "kind": "reads", "repeat": 2, "reader": "majority", "clients": 3, "perClientConcurrency": 32, "concurrency": 96, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 548853, "Errors": 0, "ThroughputPerSecond": 18295.1, "P50": 1.6405999999842606, "P95": 22.89089999999851, "P99": 27.014999999955762, "P999": 44.76540000003297, "Max": 99.85420000000158 }, "backgroundMajorityWrites": { "acknowledged": 985, "errors": {}, "perSecond": 28.142857142857142, "p50": 13.038799999980256, "p95": 30.627000000036787, "p99": 41.82860000000801, "max": 75.91539999999804 } }, { "kind": "reads", "repeat": 2, "reader": "linearizable", "clients": 10, "perClientConcurrency": 32, "concurrency": 320, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 287524, "Errors": 0, "ThroughputPerSecond": 9584.133333333333, "P50": 35.33939999999711, "P95": 45.62859999999637, "P99": 51.618900000001304, "P999": 67.20160000002943, "Max": 97.74050000001444 }, "backgroundMajorityWrites": { "acknowledged": 824, "errors": {}, "perSecond": 23.542857142857144, "p50": 22.07750000001397, "p95": 45.72840000002179, "p99": 50.917399999976624, "max": 94.0103999999701 } }, { "kind": "reads", "repeat": 2, "reader": "majority", "clients": 10, "perClientConcurrency": 32, "concurrency": 320, "hangs": 0, "overDeadline": 0, "errorsByKind": {}, "result": { "Count": 408339, "Errors": 0, "ThroughputPerSecond": 13611.3, "P50": 25.854599999962375, "P95": 37.24970000004396, "P99": 58.26860000001034, "P999": 81.68430000002263, "Max": 194.3481000000029 }, "backgroundMajorityWrites": { "acknowledged": 785, "errors": {}, "perSecond": 22.428571428571427, "p50": 29.802800000004936, "p95": 55.93459999997867, "p99": 73.58199999999488, "max": 120.16079999995418 } } ], "failover": [ { "kind": "stepdown", "reader": "linearizable", "oldPrimary": "m1", "newPrimary": "m2", "concurrency": 96, "summary": { "EventMs": 460022.7731, "Total": 523839, "Failures": 0, "OverDeadline": 0, "Hangs": 0, "MaxDurationMs": 223.09070000000065, "FirstFailureAfterEventMs": null, "LastFailureEndAfterEventMs": null, "FirstSuccessAfterLastFailureMs": 5.100299999990966, "ErrorsByKind": {} }, "recoveryAfterRestoreMs": null, "stale": { "ReadsChecked": 523839, "ReadsAfterAnAck": 260776, "Violations": 0, "Examples": [] }, "writes": { "acknowledged": 3089, "errors": {}, "perSecond": 61.40585400871317, "p50": 6.108700000040699, "p95": 23.60770000005141, "p99": 27.289099999994505, "max": 109.27500000002328 }, "timeline": [ { "second": -11, "ok": 2, "failed": 0, "maxMs": 4 }, { "second": -10, "ok": 13587, "failed": 0, "maxMs": 29 }, { "second": -9, "ok": 13774, "failed": 0, "maxMs": 26 }, { "second": -8, "ok": 10467, "failed": 0, "maxMs": 70 }, { "second": -7, "ok": 11418, "failed": 0, "maxMs": 46 }, { "second": -6, "ok": 10858, "failed": 0, "maxMs": 33 }, { "second": -5, "ok": 11618, "failed": 0, "maxMs": 30 }, { "second": -4, "ok": 11971, "failed": 0, "maxMs": 30 }, { "second": -3, "ok": 11042, "failed": 0, "maxMs": 28 }, { "second": -2, "ok": 11289, "failed": 0, "maxMs": 31 }, { "second": -1, "ok": 11226, "failed": 0, "maxMs": 145 }, { "second": 0, "ok": 8310, "failed": 0, "maxMs": 223 }, { "second": 1, "ok": 11824, "failed": 0, "maxMs": 28 }, { "second": 2, "ok": 13860, "failed": 0, "maxMs": 29 }, { "second": 3, "ok": 9757, "failed": 0, "maxMs": 33 }, { "second": 4, "ok": 9531, "failed": 0, "maxMs": 30 }, { "second": 5, "ok": 9142, "failed": 0, "maxMs": 31 }, { "second": 6, "ok": 9703, "failed": 0, "maxMs": 34 }, { "second": 7, "ok": 9570, "failed": 0, "maxMs": 31 }, { "second": 8, "ok": 9491, "failed": 0, "maxMs": 33 }, { "second": 9, "ok": 9794, "failed": 0, "maxMs": 33 }, { "second": 10, "ok": 9451, "failed": 0, "maxMs": 32 }, { "second": 11, "ok": 9979, "failed": 0, "maxMs": 52 }, { "second": 12, "ok": 10497, "failed": 0, "maxMs": 30 }, { "second": 13, "ok": 14906, "failed": 0, "maxMs": 24 }, { "second": 14, "ok": 9710, "failed": 0, "maxMs": 37 }, { "second": 15, "ok": 9520, "failed": 0, "maxMs": 31 }, { "second": 16, "ok": 9329, "failed": 0, "maxMs": 34 }, { "second": 17, "ok": 10076, "failed": 0, "maxMs": 32 }, { "second": 18, "ok": 9444, "failed": 0, "maxMs": 33 }, { "second": 19, "ok": 9943, "failed": 0, "maxMs": 36 }, { "second": 20, "ok": 9766, "failed": 0, "maxMs": 30 }, { "second": 21, "ok": 9412, "failed": 0, "maxMs": 33 }, { "second": 22, "ok": 9981, "failed": 0, "maxMs": 31 }, { "second": 23, "ok": 9661, "failed": 0, "maxMs": 33 }, { "second": 24, "ok": 14463, "failed": 0, "maxMs": 23 }, { "second": 25, "ok": 10050, "failed": 0, "maxMs": 33 }, { "second": 26, "ok": 9256, "failed": 0, "maxMs": 41 }, { "second": 27, "ok": 9721, "failed": 0, "maxMs": 32 }, { "second": 28, "ok": 9922, "failed": 0, "maxMs": 30 }, { "second": 29, "ok": 9317, "failed": 0, "maxMs": 32 }, { "second": 30, "ok": 9814, "failed": 0, "maxMs": 30 }, { "second": 31, "ok": 9110, "failed": 0, "maxMs": 32 }, { "second": 32, "ok": 9506, "failed": 0, "maxMs": 32 }, { "second": 33, "ok": 10931, "failed": 0, "maxMs": 34 }, { "second": 34, "ok": 8882, "failed": 0, "maxMs": 36 }, { "second": 35, "ok": 13259, "failed": 0, "maxMs": 28 }, { "second": 36, "ok": 10719, "failed": 0, "maxMs": 35 }, { "second": 37, "ok": 9207, "failed": 0, "maxMs": 39 }, { "second": 38, "ok": 10133, "failed": 0, "maxMs": 29 }, { "second": 39, "ok": 9505, "failed": 0, "maxMs": 34 }, { "second": 40, "ok": 135, "failed": 0, "maxMs": 30 } ] }, { "kind": "stepdown", "reader": "majority", "oldPrimary": "m2", "newPrimary": "m3", "concurrency": 96, "summary": { "EventMs": 510350.3456, "Total": 867833, "Failures": 0, "OverDeadline": 0, "Hangs": 0, "MaxDurationMs": 124.3462999999756, "FirstFailureAfterEventMs": null, "LastFailureEndAfterEventMs": null, "FirstSuccessAfterLastFailureMs": 1.4743000000016764, "ErrorsByKind": {} }, "recoveryAfterRestoreMs": null, "stale": { "ReadsChecked": 867833, "ReadsAfterAnAck": 430356, "Violations": 0, "Examples": [] }, "writes": { "acknowledged": 2288, "errors": {}, "perSecond": 45.466105367194466, "p50": 18.51300000003539, "p95": 31.804299999959767, "p99": 46.71310000005178, "max": 90.68890000000829 }, "timeline": [ { "second": -11, "ok": 194, "failed": 0, "maxMs": 22 }, { "second": -10, "ok": 16818, "failed": 0, "maxMs": 84 }, { "second": -9, "ok": 16736, "failed": 0, "maxMs": 51 }, { "second": -8, "ok": 16924, "failed": 0, "maxMs": 54 }, { "second": -7, "ok": 21337, "failed": 0, "maxMs": 54 }, { "second": -6, "ok": 18673, "failed": 0, "maxMs": 32 }, { "second": -5, "ok": 19240, "failed": 0, "maxMs": 39 }, { "second": -4, "ok": 22501, "failed": 0, "maxMs": 73 }, { "second": -3, "ok": 17567, "failed": 0, "maxMs": 59 }, { "second": -2, "ok": 18441, "failed": 0, "maxMs": 71 }, { "second": -1, "ok": 17024, "failed": 0, "maxMs": 51 }, { "second": 0, "ok": 17158, "failed": 0, "maxMs": 124 }, { "second": 1, "ok": 16383, "failed": 0, "maxMs": 58 }, { "second": 2, "ok": 16492, "failed": 0, "maxMs": 56 }, { "second": 3, "ok": 18073, "failed": 0, "maxMs": 44 }, { "second": 4, "ok": 16084, "failed": 0, "maxMs": 57 }, { "second": 5, "ok": 16425, "failed": 0, "maxMs": 64 }, { "second": 6, "ok": 16783, "failed": 0, "maxMs": 56 }, { "second": 7, "ok": 20592, "failed": 0, "maxMs": 107 }, { "second": 8, "ok": 17430, "failed": 0, "maxMs": 65 }, { "second": 9, "ok": 16996, "failed": 0, "maxMs": 42 }, { "second": 10, "ok": 16685, "failed": 0, "maxMs": 67 }, { "second": 11, "ok": 17017, "failed": 0, "maxMs": 62 }, { "second": 12, "ok": 16204, "failed": 0, "maxMs": 59 }, { "second": 13, "ok": 17882, "failed": 0, "maxMs": 57 }, { "second": 14, "ok": 17546, "failed": 0, "maxMs": 52 }, { "second": 15, "ok": 16721, "failed": 0, "maxMs": 67 }, { "second": 16, "ok": 18033, "failed": 0, "maxMs": 52 }, { "second": 17, "ok": 16532, "failed": 0, "maxMs": 57 }, { "second": 18, "ok": 21014, "failed": 0, "maxMs": 73 }, { "second": 19, "ok": 18389, "failed": 0, "maxMs": 53 }, { "second": 20, "ok": 16561, "failed": 0, "maxMs": 64 }, { "second": 21, "ok": 17441, "failed": 0, "maxMs": 33 }, { "second": 22, "ok": 17139, "failed": 0, "maxMs": 56 }, { "second": 23, "ok": 15721, "failed": 0, "maxMs": 59 }, { "second": 24, "ok": 17008, "failed": 0, "maxMs": 88 }, { "second": 25, "ok": 16512, "failed": 0, "maxMs": 58 }, { "second": 26, "ok": 17070, "failed": 0, "maxMs": 57 }, { "second": 27, "ok": 16708, "failed": 0, "maxMs": 78 }, { "second": 28, "ok": 15640, "failed": 0, "maxMs": 58 }, { "second": 29, "ok": 19539, "failed": 0, "maxMs": 100 }, { "second": 30, "ok": 17006, "failed": 0, "maxMs": 95 }, { "second": 31, "ok": 15596, "failed": 0, "maxMs": 74 }, { "second": 32, "ok": 18482, "failed": 0, "maxMs": 67 }, { "second": 33, "ok": 16159, "failed": 0, "maxMs": 121 }, { "second": 34, "ok": 17320, "failed": 0, "maxMs": 60 }, { "second": 35, "ok": 15249, "failed": 0, "maxMs": 77 }, { "second": 36, "ok": 15492, "failed": 0, "maxMs": 58 }, { "second": 37, "ok": 16821, "failed": 0, "maxMs": 59 }, { "second": 38, "ok": 15833, "failed": 0, "maxMs": 59 }, { "second": 39, "ok": 16638, "failed": 0, "maxMs": 84 }, { "second": 40, "ok": 4, "failed": 0, "maxMs": 1 } ] }, { "kind": "kill", "reader": "linearizable", "oldPrimary": "m3", "newPrimary": "m2", "concurrency": 96, "summary": { "EventMs": 560684.5755, "Total": 257435, "Failures": 192, "OverDeadline": 0, "Hangs": 0, "MaxDurationMs": 5045.136599999969, "FirstFailureAfterEventMs": 86.76899999997113, "LastFailureEndAfterEventMs": 10191.17949999997, "FirstSuccessAfterLastFailureMs": 10972.539400000009, "ErrorsByKind": { "ConnectionError": 96, "ClientDeadline": 96 } }, "recoveryAfterRestoreMs": null, "stale": { "ReadsChecked": 257243, "ReadsAfterAnAck": 126643, "Violations": 0, "Examples": [] }, "writes": { "acknowledged": 2549, "errors": { "ConnectionError": 2, "ClientDeadline": 2 }, "perSecond": 49.96497786322148, "p50": 5.811699999962002, "p95": 24.666299999924377, "p99": 28.20559999998659, "max": 714.5544000000227 }, "timeline": [ { "second": -11, "ok": 96, "failed": 0, "maxMs": 11 }, { "second": -10, "ok": 13948, "failed": 0, "maxMs": 27 }, { "second": -9, "ok": 8480, "failed": 0, "maxMs": 41 }, { "second": -8, "ok": 9344, "failed": 0, "maxMs": 35 }, { "second": -7, "ok": 8912, "failed": 0, "maxMs": 41 }, { "second": -6, "ok": 8699, "failed": 0, "maxMs": 33 }, { "second": -5, "ok": 8917, "failed": 0, "maxMs": 39 }, { "second": -4, "ok": 9698, "failed": 0, "maxMs": 31 }, { "second": -3, "ok": 9567, "failed": 0, "maxMs": 31 }, { "second": -2, "ok": 9218, "failed": 0, "maxMs": 35 }, { "second": -1, "ok": 8314, "failed": 0, "maxMs": 35 }, { "second": 0, "ok": 984, "failed": 96, "maxMs": 5030 }, { "second": 5, "ok": 0, "failed": 96, "maxMs": 5045 }, { "second": 10, "ok": 192, "failed": 0, "maxMs": 806 }, { "second": 11, "ok": 4767, "failed": 0, "maxMs": 56 }, { "second": 12, "ok": 5567, "failed": 0, "maxMs": 43 }, { "second": 13, "ok": 5935, "failed": 0, "maxMs": 47 }, { "second": 14, "ok": 5737, "failed": 0, "maxMs": 44 }, { "second": 15, "ok": 5940, "failed": 0, "maxMs": 50 }, { "second": 16, "ok": 5860, "failed": 0, "maxMs": 42 }, { "second": 17, "ok": 5542, "failed": 0, "maxMs": 45 }, { "second": 18, "ok": 6060, "failed": 0, "maxMs": 49 }, { "second": 19, "ok": 4570, "failed": 0, "maxMs": 51 }, { "second": 20, "ok": 6070, "failed": 0, "maxMs": 41 }, { "second": 21, "ok": 5859, "failed": 0, "maxMs": 42 }, { "second": 22, "ok": 5514, "failed": 0, "maxMs": 54 }, { "second": 23, "ok": 6669, "failed": 0, "maxMs": 79 }, { "second": 24, "ok": 5922, "failed": 0, "maxMs": 49 }, { "second": 25, "ok": 4389, "failed": 0, "maxMs": 49 }, { "second": 26, "ok": 3428, "failed": 0, "maxMs": 50 }, { "second": 27, "ok": 5705, "failed": 0, "maxMs": 47 }, { "second": 28, "ok": 6167, "failed": 0, "maxMs": 43 }, { "second": 29, "ok": 5840, "failed": 0, "maxMs": 44 }, { "second": 30, "ok": 6030, "failed": 0, "maxMs": 41 }, { "second": 31, "ok": 6130, "failed": 0, "maxMs": 42 }, { "second": 32, "ok": 5983, "failed": 0, "maxMs": 42 }, { "second": 33, "ok": 6188, "failed": 0, "maxMs": 41 }, { "second": 34, "ok": 6161, "failed": 0, "maxMs": 41 }, { "second": 35, "ok": 4844, "failed": 0, "maxMs": 48 }, { "second": 36, "ok": 3328, "failed": 0, "maxMs": 51 }, { "second": 37, "ok": 4258, "failed": 0, "maxMs": 51 }, { "second": 38, "ok": 5124, "failed": 0, "maxMs": 43 }, { "second": 39, "ok": 5747, "failed": 0, "maxMs": 50 }, { "second": 40, "ok": 1540, "failed": 0, "maxMs": 48 } ] }, { "kind": "kill", "reader": "majority", "oldPrimary": "m2", "newPrimary": "m3", "concurrency": 96, "summary": { "EventMs": 615730.8889, "Total": 728980, "Failures": 192, "OverDeadline": 0, "Hangs": 0, "MaxDurationMs": 5040.880899999989, "FirstFailureAfterEventMs": 97.0564000000013, "LastFailureEndAfterEventMs": 10177.220899999957, "FirstSuccessAfterLastFailureMs": 10947.850499999942, "ErrorsByKind": { "ConnectionError": 96, "ClientDeadline": 96 } }, "recoveryAfterRestoreMs": null, "stale": { "ReadsChecked": 728788, "ReadsAfterAnAck": 360517, "Violations": 0, "Examples": [] }, "writes": { "acknowledged": 1897, "errors": { "ConnectionError": 2, "ClientDeadline": 2 }, "perSecond": 35.72537937410307, "p50": 17.211899999994785, "p95": 30.37650000001304, "p99": 36.60430000000633, "max": 733.3172999999952 }, "timeline": [ { "second": -11, "ok": 61, "failed": 0, "maxMs": 29 }, { "second": -10, "ok": 21011, "failed": 0, "maxMs": 78 }, { "second": -9, "ok": 17019, "failed": 0, "maxMs": 54 }, { "second": -8, "ok": 18433, "failed": 0, "maxMs": 41 }, { "second": -7, "ok": 16821, "failed": 0, "maxMs": 56 }, { "second": -6, "ok": 19359, "failed": 0, "maxMs": 53 }, { "second": -5, "ok": 18104, "failed": 0, "maxMs": 53 }, { "second": -4, "ok": 19297, "failed": 0, "maxMs": 51 }, { "second": -3, "ok": 16428, "failed": 0, "maxMs": 52 }, { "second": -2, "ok": 16341, "failed": 0, "maxMs": 55 }, { "second": -1, "ok": 18713, "failed": 0, "maxMs": 46 }, { "second": 0, "ok": 1863, "failed": 96, "maxMs": 5040 }, { "second": 5, "ok": 0, "failed": 96, "maxMs": 5041 }, { "second": 10, "ok": 481, "failed": 0, "maxMs": 870 }, { "second": 11, "ok": 18629, "failed": 0, "maxMs": 101 }, { "second": 12, "ok": 22554, "failed": 0, "maxMs": 62 }, { "second": 13, "ok": 17072, "failed": 0, "maxMs": 56 }, { "second": 14, "ok": 19029, "failed": 0, "maxMs": 58 }, { "second": 15, "ok": 19640, "failed": 0, "maxMs": 47 }, { "second": 16, "ok": 18158, "failed": 0, "maxMs": 56 }, { "second": 17, "ok": 17919, "failed": 0, "maxMs": 56 }, { "second": 18, "ok": 18909, "failed": 0, "maxMs": 51 }, { "second": 19, "ok": 20316, "failed": 0, "maxMs": 46 }, { "second": 20, "ok": 17020, "failed": 0, "maxMs": 51 }, { "second": 21, "ok": 17574, "failed": 0, "maxMs": 52 }, { "second": 22, "ok": 17527, "failed": 0, "maxMs": 51 }, { "second": 23, "ok": 21525, "failed": 0, "maxMs": 69 }, { "second": 24, "ok": 18070, "failed": 0, "maxMs": 54 }, { "second": 25, "ok": 16988, "failed": 0, "maxMs": 42 }, { "second": 26, "ok": 16705, "failed": 0, "maxMs": 63 }, { "second": 27, "ok": 18128, "failed": 0, "maxMs": 47 }, { "second": 28, "ok": 17401, "failed": 0, "maxMs": 33 }, { "second": 29, "ok": 17736, "failed": 0, "maxMs": 54 }, { "second": 30, "ok": 17528, "failed": 0, "maxMs": 46 }, { "second": 31, "ok": 17952, "failed": 0, "maxMs": 45 }, { "second": 32, "ok": 18008, "failed": 0, "maxMs": 56 }, { "second": 33, "ok": 17331, "failed": 0, "maxMs": 46 }, { "second": 34, "ok": 21667, "failed": 0, "maxMs": 85 }, { "second": 35, "ok": 19821, "failed": 0, "maxMs": 56 }, { "second": 36, "ok": 17389, "failed": 0, "maxMs": 63 }, { "second": 37, "ok": 19170, "failed": 0, "maxMs": 45 }, { "second": 38, "ok": 18814, "failed": 0, "maxMs": 39 }, { "second": 39, "ok": 18352, "failed": 0, "maxMs": 46 }, { "second": 40, "ok": 7925, "failed": 0, "maxMs": 57 } ] }, { "kind": "partition", "reader": "linearizable", "oldPrimary": "m3", "newPrimary": "m1", "concurrency": 96, "summary": { "EventMs": 672868.0856, "Total": 172378, "Failures": 192, "OverDeadline": 0, "Hangs": 0, "MaxDurationMs": 5068.520200000028, "FirstFailureAfterEventMs": 40.15060000005178, "LastFailureEndAfterEventMs": 10119.188300000038, "FirstSuccessAfterLastFailureMs": 12078.606099999975, "ErrorsByKind": { "ClientDeadline": 132, "MongoConnectionPoolPausedException": 56, "ConnectionError": 4 } }, "recoveryAfterRestoreMs": null, "stale": { "ReadsChecked": 172186, "ReadsAfterAnAck": 84279, "Violations": 0, "Examples": [] }, "writes": { "acknowledged": 1338, "errors": { "ClientDeadline": 4 }, "perSecond": 26.53354789653125, "p50": 5.858699999982491, "p95": 24.56479999993462, "p99": 1977.5396999999648, "max": 2005.189500000095 }, "timeline": [ { "second": -11, "ok": 4, "failed": 0, "maxMs": 5 }, { "second": -10, "ok": 10306, "failed": 0, "maxMs": 67 }, { "second": -9, "ok": 9648, "failed": 0, "maxMs": 53 }, { "second": -8, "ok": 10753, "failed": 0, "maxMs": 33 }, { "second": -7, "ok": 10807, "failed": 0, "maxMs": 29 }, { "second": -6, "ok": 10730, "failed": 0, "maxMs": 34 }, { "second": -5, "ok": 10715, "failed": 0, "maxMs": 34 }, { "second": -4, "ok": 10791, "failed": 0, "maxMs": 30 }, { "second": -3, "ok": 10612, "failed": 0, "maxMs": 32 }, { "second": -2, "ok": 11021, "failed": 0, "maxMs": 31 }, { "second": -1, "ok": 16287, "failed": 0, "maxMs": 26 }, { "second": 0, "ok": 748, "failed": 96, "maxMs": 5069 }, { "second": 5, "ok": 0, "failed": 96, "maxMs": 5009 }, { "second": 10, "ok": 96, "failed": 0, "maxMs": 1992 }, { "second": 12, "ok": 98, "failed": 0, "maxMs": 2003 }, { "second": 14, "ok": 118, "failed": 0, "maxMs": 2005 }, { "second": 16, "ok": 102, "failed": 0, "maxMs": 2003 }, { "second": 18, "ok": 103, "failed": 0, "maxMs": 2002 }, { "second": 20, "ok": 96, "failed": 0, "maxMs": 2002 }, { "second": 22, "ok": 96, "failed": 0, "maxMs": 2029 }, { "second": 24, "ok": 140, "failed": 0, "maxMs": 2002 }, { "second": 26, "ok": 101, "failed": 0, "maxMs": 2002 }, { "second": 28, "ok": 101, "failed": 0, "maxMs": 1986 }, { "second": 30, "ok": 5049, "failed": 0, "maxMs": 48 }, { "second": 31, "ok": 6038, "failed": 0, "maxMs": 49 }, { "second": 32, "ok": 6390, "failed": 0, "maxMs": 45 }, { "second": 33, "ok": 5557, "failed": 0, "maxMs": 50 }, { "second": 34, "ok": 6125, "failed": 0, "maxMs": 44 }, { "second": 35, "ok": 6095, "failed": 0, "maxMs": 39 }, { "second": 36, "ok": 5883, "failed": 0, "maxMs": 46 }, { "second": 37, "ok": 5114, "failed": 0, "maxMs": 48 }, { "second": 38, "ok": 5911, "failed": 0, "maxMs": 40 }, { "second": 39, "ok": 5736, "failed": 0, "maxMs": 44 }, { "second": 40, "ok": 815, "failed": 0, "maxMs": 37 } ] }, { "kind": "partition", "reader": "majority", "oldPrimary": "m1", "newPrimary": "m2", "concurrency": 96, "summary": { "EventMs": 725318.6106, "Total": 694497, "Failures": 192, "OverDeadline": 0, "Hangs": 0, "MaxDurationMs": 5043.984300000011, "FirstFailureAfterEventMs": 45.48869999998715, "LastFailureEndAfterEventMs": 10135.215399999986, "FirstSuccessAfterLastFailureMs": 11470.7267, "ErrorsByKind": { "ClientDeadline": 187, "ConnectionError": 3, "MongoConnectionPoolPausedException": 2 } }, "recoveryAfterRestoreMs": null, "stale": { "ReadsChecked": 694305, "ReadsAfterAnAck": 344560, "Violations": 0, "Examples": [] }, "writes": { "acknowledged": 1817, "errors": { "ClientDeadline": 4 }, "perSecond": 34.575999118355774, "p50": 17.228599999914877, "p95": 31.505200000014156, "p99": 48.475800000014715, "max": 1317.8485999999102 }, "timeline": [ { "second": -11, "ok": 347, "failed": 0, "maxMs": 31 }, { "second": -10, "ok": 14382, "failed": 0, "maxMs": 76 }, { "second": -9, "ok": 20313, "failed": 0, "maxMs": 81 }, { "second": -8, "ok": 17649, "failed": 0, "maxMs": 58 }, { "second": -7, "ok": 18094, "failed": 0, "maxMs": 47 }, { "second": -6, "ok": 18198, "failed": 0, "maxMs": 49 }, { "second": -5, "ok": 16355, "failed": 0, "maxMs": 56 }, { "second": -4, "ok": 18068, "failed": 0, "maxMs": 40 }, { "second": -3, "ok": 18962, "failed": 0, "maxMs": 50 }, { "second": -2, "ok": 18400, "failed": 0, "maxMs": 50 }, { "second": -1, "ok": 18737, "failed": 0, "maxMs": 51 }, { "second": 0, "ok": 1437, "failed": 96, "maxMs": 5044 }, { "second": 5, "ok": 0, "failed": 96, "maxMs": 5029 }, { "second": 10, "ok": 96, "failed": 0, "maxMs": 1458 }, { "second": 11, "ok": 10018, "failed": 0, "maxMs": 99 }, { "second": 12, "ok": 18212, "failed": 0, "maxMs": 73 }, { "second": 13, "ok": 19499, "failed": 0, "maxMs": 71 }, { "second": 14, "ok": 19029, "failed": 0, "maxMs": 39 }, { "second": 15, "ok": 17786, "failed": 0, "maxMs": 59 }, { "second": 16, "ok": 18319, "failed": 0, "maxMs": 47 }, { "second": 17, "ok": 17953, "failed": 0, "maxMs": 52 }, { "second": 18, "ok": 17698, "failed": 0, "maxMs": 87 }, { "second": 19, "ok": 18716, "failed": 0, "maxMs": 44 }, { "second": 20, "ok": 18757, "failed": 0, "maxMs": 37 }, { "second": 21, "ok": 17087, "failed": 0, "maxMs": 60 }, { "second": 22, "ok": 18205, "failed": 0, "maxMs": 44 }, { "second": 23, "ok": 18120, "failed": 0, "maxMs": 93 }, { "second": 24, "ok": 20359, "failed": 0, "maxMs": 103 }, { "second": 25, "ok": 18270, "failed": 0, "maxMs": 46 }, { "second": 26, "ok": 17419, "failed": 0, "maxMs": 59 }, { "second": 27, "ok": 17094, "failed": 0, "maxMs": 52 }, { "second": 28, "ok": 17491, "failed": 0, "maxMs": 35 }, { "second": 29, "ok": 16333, "failed": 0, "maxMs": 55 }, { "second": 30, "ok": 15889, "failed": 0, "maxMs": 70 }, { "second": 31, "ok": 17648, "failed": 0, "maxMs": 69 }, { "second": 32, "ok": 17643, "failed": 0, "maxMs": 48 }, { "second": 33, "ok": 17714, "failed": 0, "maxMs": 61 }, { "second": 34, "ok": 17339, "failed": 0, "maxMs": 79 }, { "second": 35, "ok": 20777, "failed": 0, "maxMs": 89 }, { "second": 36, "ok": 17398, "failed": 0, "maxMs": 91 }, { "second": 37, "ok": 17188, "failed": 0, "maxMs": 56 }, { "second": 38, "ok": 16227, "failed": 0, "maxMs": 53 }, { "second": 39, "ok": 16191, "failed": 0, "maxMs": 54 }, { "second": 40, "ok": 2888, "failed": 0, "maxMs": 29 } ] }, { "kind": "nomajority", "reader": "linearizable", "oldPrimary": "m2", "newPrimary": "m3", "concurrency": 96, "summary": { "EventMs": 779104.604, "Total": 577150, "Failures": 384, "OverDeadline": 0, "Hangs": 0, "MaxDurationMs": 5037.759199999971, "FirstFailureAfterEventMs": 185.22789999993984, "LastFailureEndAfterEventMs": 20332.818299999926, "FirstSuccessAfterLastFailureMs": 20445.773699999903, "ErrorsByKind": { "ClientDeadline": 288, "NodeIsRecovering(InterruptedDueToReplStateChange)": 96 } }, "recoveryAfterRestoreMs": 108.69069999991916, "stale": { "ReadsChecked": 576766, "ReadsAfterAnAck": 287813, "Violations": 0, "Examples": [] }, "writes": { "acknowledged": 3138, "errors": { "ClientDeadline": 5, "ArgumentNullException": 2 }, "perSecond": 44.38680848324588, "p50": 5.851100000087172, "p95": 15.9081999999471, "p99": 23.626200000056997, "max": 5000.218499999959 }, "timeline": [ { "second": -10, "ok": 9493, "failed": 0, "maxMs": 36 }, { "second": -9, "ok": 8772, "failed": 0, "maxMs": 52 }, { "second": -8, "ok": 12739, "failed": 0, "maxMs": 36 }, { "second": -7, "ok": 10757, "failed": 0, "maxMs": 33 }, { "second": -6, "ok": 9220, "failed": 0, "maxMs": 33 }, { "second": -5, "ok": 9866, "failed": 0, "maxMs": 32 }, { "second": -4, "ok": 9225, "failed": 0, "maxMs": 38 }, { "second": -3, "ok": 8895, "failed": 0, "maxMs": 39 }, { "second": -2, "ok": 9505, "failed": 0, "maxMs": 33 }, { "second": -1, "ok": 9143, "failed": 0, "maxMs": 28 }, { "second": 0, "ok": 1278, "failed": 96, "maxMs": 5027 }, { "second": 5, "ok": 0, "failed": 96, "maxMs": 5038 }, { "second": 10, "ok": 0, "failed": 96, "maxMs": 5019 }, { "second": 15, "ok": 0, "failed": 96, "maxMs": 5032 }, { "second": 20, "ok": 5242, "failed": 0, "maxMs": 246 }, { "second": 21, "ok": 11997, "failed": 0, "maxMs": 32 }, { "second": 22, "ok": 12037, "failed": 0, "maxMs": 31 }, { "second": 23, "ok": 11676, "failed": 0, "maxMs": 29 }, { "second": 24, "ok": 10533, "failed": 0, "maxMs": 36 }, { "second": 25, "ok": 15405, "failed": 0, "maxMs": 22 }, { "second": 26, "ok": 12483, "failed": 0, "maxMs": 36 }, { "second": 27, "ok": 11381, "failed": 0, "maxMs": 31 }, { "second": 28, "ok": 12422, "failed": 0, "maxMs": 26 }, { "second": 29, "ok": 10341, "failed": 0, "maxMs": 30 }, { "second": 30, "ok": 11425, "failed": 0, "maxMs": 30 }, { "second": 31, "ok": 11950, "failed": 0, "maxMs": 27 }, { "second": 32, "ok": 11718, "failed": 0, "maxMs": 32 }, { "second": 33, "ok": 11850, "failed": 0, "maxMs": 31 }, { "second": 34, "ok": 10704, "failed": 0, "maxMs": 31 }, { "second": 35, "ok": 11043, "failed": 0, "maxMs": 33 }, { "second": 36, "ok": 14852, "failed": 0, "maxMs": 35 }, { "second": 37, "ok": 12778, "failed": 0, "maxMs": 30 }, { "second": 38, "ok": 11314, "failed": 0, "maxMs": 39 }, { "second": 39, "ok": 11330, "failed": 0, "maxMs": 27 }, { "second": 40, "ok": 10251, "failed": 0, "maxMs": 32 }, { "second": 41, "ok": 11537, "failed": 0, "maxMs": 30 }, { "second": 42, "ok": 12304, "failed": 0, "maxMs": 29 }, { "second": 43, "ok": 11379, "failed": 0, "maxMs": 27 }, { "second": 44, "ok": 12046, "failed": 0, "maxMs": 29 }, { "second": 45, "ok": 10569, "failed": 0, "maxMs": 34 }, { "second": 46, "ok": 11259, "failed": 0, "maxMs": 33 }, { "second": 47, "ok": 15213, "failed": 0, "maxMs": 27 }, { "second": 48, "ok": 13482, "failed": 0, "maxMs": 32 }, { "second": 49, "ok": 12492, "failed": 0, "maxMs": 31 }, { "second": 50, "ok": 10979, "failed": 0, "maxMs": 33 }, { "second": 51, "ok": 11672, "failed": 0, "maxMs": 32 }, { "second": 52, "ok": 11851, "failed": 0, "maxMs": 30 }, { "second": 53, "ok": 11767, "failed": 0, "maxMs": 30 }, { "second": 54, "ok": 12147, "failed": 0, "maxMs": 27 }, { "second": 55, "ok": 10989, "failed": 0, "maxMs": 30 }, { "second": 56, "ok": 10550, "failed": 0, "maxMs": 35 }, { "second": 57, "ok": 11618, "failed": 0, "maxMs": 31 }, { "second": 58, "ok": 13747, "failed": 0, "maxMs": 29 }, { "second": 59, "ok": 14353, "failed": 0, "maxMs": 26 }, { "second": 60, "ok": 5187, "failed": 0, "maxMs": 36 } ] }, { "kind": "nomajority", "reader": "majority", "oldPrimary": "m3", "newPrimary": "m3", "concurrency": 96, "summary": { "EventMs": 849814.2678, "Total": 1012173, "Failures": 192, "OverDeadline": 0, "Hangs": 0, "MaxDurationMs": 5023.074999999953, "FirstFailureAfterEventMs": 10200.205699999933, "LastFailureEndAfterEventMs": 20245.312599999947, "FirstSuccessAfterLastFailureMs": 21383.921399999992, "ErrorsByKind": { "ClientDeadline": 149, "NotWritablePrimary": 43 } }, "recoveryAfterRestoreMs": null, "stale": { "ReadsChecked": 1011981, "ReadsAfterAnAck": 502719, "Violations": 0, "Examples": [] }, "writes": { "acknowledged": 2292, "errors": { "ClientDeadline": 6, "ArgumentNullException": 2 }, "perSecond": 32.33295686739962, "p50": 18.631400000071153, "p95": 30.975600000005215, "p99": 41.51340000005439, "max": 823.6518999999389 }, "timeline": [ { "second": -11, "ok": 188, "failed": 0, "maxMs": 45 }, { "second": -10, "ok": 15275, "failed": 0, "maxMs": 47 }, { "second": -9, "ok": 16100, "failed": 0, "maxMs": 50 }, { "second": -8, "ok": 15954, "failed": 0, "maxMs": 54 }, { "second": -7, "ok": 17444, "failed": 0, "maxMs": 49 }, { "second": -6, "ok": 17479, "failed": 0, "maxMs": 53 }, { "second": -5, "ok": 16992, "failed": 0, "maxMs": 74 }, { "second": -4, "ok": 17302, "failed": 0, "maxMs": 69 }, { "second": -3, "ok": 16095, "failed": 0, "maxMs": 60 }, { "second": -2, "ok": 16157, "failed": 0, "maxMs": 94 }, { "second": -1, "ok": 23341, "failed": 0, "maxMs": 98 }, { "second": 0, "ok": 17397, "failed": 0, "maxMs": 75 }, { "second": 1, "ok": 15738, "failed": 0, "maxMs": 88 }, { "second": 2, "ok": 16383, "failed": 0, "maxMs": 56 }, { "second": 3, "ok": 15569, "failed": 0, "maxMs": 61 }, { "second": 4, "ok": 14711, "failed": 0, "maxMs": 56 }, { "second": 5, "ok": 14684, "failed": 0, "maxMs": 79 }, { "second": 6, "ok": 15759, "failed": 0, "maxMs": 54 }, { "second": 7, "ok": 17435, "failed": 0, "maxMs": 76 }, { "second": 8, "ok": 13871, "failed": 0, "maxMs": 110 }, { "second": 9, "ok": 16497, "failed": 0, "maxMs": 48 }, { "second": 10, "ok": 4361, "failed": 96, "maxMs": 5005 }, { "second": 15, "ok": 0, "failed": 96, "maxMs": 5023 }, { "second": 20, "ok": 96, "failed": 0, "maxMs": 1145 }, { "second": 21, "ok": 12999, "failed": 0, "maxMs": 66 }, { "second": 22, "ok": 15312, "failed": 0, "maxMs": 56 }, { "second": 23, "ok": 16744, "failed": 0, "maxMs": 55 }, { "second": 24, "ok": 16192, "failed": 0, "maxMs": 58 }, { "second": 25, "ok": 17801, "failed": 0, "maxMs": 54 }, { "second": 26, "ok": 15831, "failed": 0, "maxMs": 71 }, { "second": 27, "ok": 16818, "failed": 0, "maxMs": 67 }, { "second": 28, "ok": 16100, "failed": 0, "maxMs": 66 }, { "second": 29, "ok": 17075, "failed": 0, "maxMs": 53 }, { "second": 30, "ok": 19914, "failed": 0, "maxMs": 50 }, { "second": 31, "ok": 17331, "failed": 0, "maxMs": 51 }, { "second": 32, "ok": 22643, "failed": 0, "maxMs": 85 }, { "second": 33, "ok": 16119, "failed": 0, "maxMs": 81 }, { "second": 34, "ok": 16355, "failed": 0, "maxMs": 63 }, { "second": 35, "ok": 18014, "failed": 0, "maxMs": 70 }, { "second": 36, "ok": 15745, "failed": 0, "maxMs": 67 }, { "second": 37, "ok": 16113, "failed": 0, "maxMs": 64 }, { "second": 38, "ok": 16912, "failed": 0, "maxMs": 54 }, { "second": 39, "ok": 18246, "failed": 0, "maxMs": 91 }, { "second": 40, "ok": 17516, "failed": 0, "maxMs": 54 }, { "second": 41, "ok": 16565, "failed": 0, "maxMs": 52 }, { "second": 42, "ok": 16147, "failed": 0, "maxMs": 59 }, { "second": 43, "ok": 20884, "failed": 0, "maxMs": 102 }, { "second": 44, "ok": 19179, "failed": 0, "maxMs": 57 }, { "second": 45, "ok": 15115, "failed": 0, "maxMs": 60 }, { "second": 46, "ok": 18484, "failed": 0, "maxMs": 47 }, { "second": 47, "ok": 16235, "failed": 0, "maxMs": 58 }, { "second": 48, "ok": 18071, "failed": 0, "maxMs": 48 }, { "second": 49, "ok": 16165, "failed": 0, "maxMs": 52 }, { "second": 50, "ok": 17180, "failed": 0, "maxMs": 48 }, { "second": 51, "ok": 17576, "failed": 0, "maxMs": 53 }, { "second": 52, "ok": 16479, "failed": 0, "maxMs": 74 }, { "second": 53, "ok": 16915, "failed": 0, "maxMs": 54 }, { "second": 54, "ok": 20573, "failed": 0, "maxMs": 70 }, { "second": 55, "ok": 16978, "failed": 0, "maxMs": 51 }, { "second": 56, "ok": 17394, "failed": 0, "maxMs": 62 }, { "second": 57, "ok": 16322, "failed": 0, "maxMs": 51 }, { "second": 58, "ok": 16984, "failed": 0, "maxMs": 42 }, { "second": 59, "ok": 16596, "failed": 0, "maxMs": 67 }, { "second": 60, "ok": 7531, "failed": 0, "maxMs": 46 } ] } ], "errorExamples": { "ConnectionError": "MongoDB.Driver.MongoConnectionException: An exception occurred while receiving a message from the server.\n ---> System.IO.IOException: Unable to read data from the transport connection: Connection reset by peer.\n ---> System.Net.Sockets.SocketException (104): Connection reset by peer\n --- End of inner exception stack trace ---\n at System.Net.Sockets.Socket.AwaitableSocketAsyncEventArgs.ThrowException(SocketError error, CancellationToken cancellationToken)\n at System.Threading.Tasks.ValueTask`1.ValueTaskSourceAsTask.<>c.<.cctor>b__4_0(Object state)", "ArgumentNullException": "System.ArgumentNullException: Value cannot be null. (Parameter 'connectionId')\n at MongoDB.Driver.Core.Misc.Ensure.IsNotNull[T](T value, String paramName)\n at MongoDB.Driver.MongoServerException..ctor(ConnectionId connectionId, String message, Exception innerException)\n at MongoDB.Driver.MongoServerException..ctor(ConnectionId connectionId, String message)\n at MongoDB.Driver.Core.Operations.MongoBulkWriteOperationException..ctor(ConnectionId connectionId, BulkWriteOperationResult result, IReadOnlyList`1 writeErrors, BulkWriteConcernError writeConcernError, IReadOnlyList`1 unprocessedRequests)\n at MongoDB.Driver.Core.Operations.BulkWriteBatchResultCombiner.CreateBulkWriteException(ConnectionId connectionId, IEnumerable`1 remainingRequests)", "NotWritablePrimary": "MongoDB.Driver.MongoNotPrimaryException: Server returned not primary error.\n at MongoDB.Driver.Core.Operations.RetryableReadOperationExecutor.ExecuteAsync[TResult](OperationContext operationContext, IRetryableReadOperation`1 operation, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.ReadCommandOperation`1.ExecuteAsync(OperationContext operationContext, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.FindOperation`1.ExecuteAsync(OperationContext operationContext, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.FindOperation`1.ExecuteAsync(OperationContext operationContext, IReadBinding binding)\n at MongoDB.Driver.OperationExecutor.ExecuteReadOperationAsync[TResult](OperationContext operationContext, IClientSessionHandle session, IReadOperation`1 operation, ReadPreference readPreference, Boolean allowChannelPinning)", "MongoConnectionPoolPausedException": "MongoDB.Driver.MongoConnectionPoolPausedException: The connection pool is in paused state for server rs-m3:27017.\n at MongoDB.Driver.Core.Operations.RetryableReadOperationExecutor.ExecuteAsync[TResult](OperationContext operationContext, IRetryableReadOperation`1 operation, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.ReadCommandOperation`1.ExecuteAsync(OperationContext operationContext, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.FindOperation`1.ExecuteAsync(OperationContext operationContext, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.FindOperation`1.ExecuteAsync(OperationContext operationContext, IReadBinding binding)\n at MongoDB.Driver.OperationExecutor.ExecuteReadOperationAsync[TResult](OperationContext operationContext, IClientSessionHandle session, IReadOperation`1 operation, ReadPreference readPreference, Boolean allowChannelPinning)", "NodeIsRecovering(InterruptedDueToReplStateChange)": "MongoDB.Driver.MongoNodeIsRecoveringException: Server returned node is recovering error (code = 11602, codeName = \"InterruptedDueToReplStateChange\").\n at MongoDB.Driver.Core.Operations.RetryableReadOperationExecutor.ExecuteAsync[TResult](OperationContext operationContext, IRetryableReadOperation`1 operation, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.ReadCommandOperation`1.ExecuteAsync(OperationContext operationContext, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.FindOperation`1.ExecuteAsync(OperationContext operationContext, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.FindOperation`1.ExecuteAsync(OperationContext operationContext, IReadBinding binding)\n at MongoDB.Driver.OperationExecutor.ExecuteReadOperationAsync[TResult](OperationContext operationContext, IClientSessionHandle session, IReadOperation`1 operation, ReadPreference readPreference, Boolean allowChannelPinning)", "ClientDeadline": "System.OperationCanceledException: The operation was canceled.\n at MongoDB.Driver.Core.Operations.RetryableReadOperationExecutor.ExecuteAsync[TResult](OperationContext operationContext, IRetryableReadOperation`1 operation, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.ReadCommandOperation`1.ExecuteAsync(OperationContext operationContext, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.FindOperation`1.ExecuteAsync(OperationContext operationContext, RetryableReadContext context)\n at MongoDB.Driver.Core.Operations.FindOperation`1.ExecuteAsync(OperationContext operationContext, IReadBinding binding)\n at MongoDB.Driver.OperationExecutor.ExecuteReadOperationAsync[TResult](OperationContext operationContext, IClientSessionHandle session, IReadOperation`1 operation, ReadPreference readPreference, Boolean allowChannelPinning)" }, "dualPrimary": [ { "iteration": 1, "stalePrimary": "m3", "newPrimary": "m1", "grantedRevision": 757, "readOnPBeforePartition": { "StartMs": 910747.3785, "EndMs": 910754.6512, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 757, "DurationMs": 7.272699999972247 }, "stepUp": "ok", "newPrimaryAfterMs": 197.92510000010952, "revokeAttempts": 1, "revokedRevision": 758, "revokeAckAfterPartitionMs": 10155.69020000007, "stalePrimaryStillWritablePrimary": true, "linearizableOnStalePrimary": { "StartMs": 920911.2513, "EndMs": 925912.2048, "Ok": false, "Error": "ClientDeadline", "DocIndex": 0, "Revision": -1, "DurationMs": 5000.953499999945 }, "linearizableOnStalePrimaryHung": false, "majorityOnStalePrimary": { "StartMs": 920911.4786, "EndMs": 920914.0989, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 757, "DurationMs": 2.620299999951385 }, "localOnStalePrimary": { "StartMs": 920912.1302, "EndMs": 920914.0771, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 757, "DurationMs": 1.9468999999808148 }, "linearizableViaReplicaSet": { "StartMs": 925913.8966, "EndMs": 926911.2043, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 758, "DurationMs": 997.3077000000048 }, "linearizableSafe": true, "linearizableBounded": true, "conclusive": true, "majorityReturnedRevokedRole": true }, { "iteration": 2, "stalePrimary": "m1", "newPrimary": "m2", "grantedRevision": 759, "readOnPBeforePartition": { "StartMs": 929075.7231, "EndMs": 929083.844, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 759, "DurationMs": 8.120900000096299 }, "stepUp": "ok", "newPrimaryAfterMs": 10191.832799999975, "revokeAttempts": 1, "revokedRevision": 760, "revokeAckAfterPartitionMs": 10196.135099999956, "stalePrimaryStillWritablePrimary": true, "linearizableOnStalePrimary": { "StartMs": 939280.6826, "EndMs": 944281.8628, "Ok": false, "Error": "ClientDeadline", "DocIndex": 0, "Revision": -1, "DurationMs": 5001.180200000061 }, "linearizableOnStalePrimaryHung": false, "majorityOnStalePrimary": { "StartMs": 939280.8537, "EndMs": 939283.0919, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 759, "DurationMs": 2.2382000000216067 }, "localOnStalePrimary": { "StartMs": 939280.9343, "EndMs": 939283.204, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 759, "DurationMs": 2.269700000062585 }, "linearizableViaReplicaSet": { "StartMs": 944283.2579, "EndMs": 944377.7466, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 760, "DurationMs": 94.48869999998715 }, "linearizableSafe": true, "linearizableBounded": true, "conclusive": true, "majorityReturnedRevokedRole": true }, { "iteration": 3, "stalePrimary": "m2", "newPrimary": "m1", "grantedRevision": 761, "readOnPBeforePartition": { "StartMs": 946042.4293, "EndMs": 946051.7006, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 761, "DurationMs": 9.271300000022165 }, "stepUp": "ok", "newPrimaryAfterMs": 220.21100000001024, "revokeAttempts": 3, "revokedRevision": 762, "revokeAckAfterPartitionMs": 22177.87710000004, "stalePrimaryStillWritablePrimary": true, "linearizableOnStalePrimary": { "StartMs": 968230.4228, "EndMs": 973232.1881, "Ok": false, "Error": "ClientDeadline", "DocIndex": 0, "Revision": -1, "DurationMs": 5001.765300000086 }, "linearizableOnStalePrimaryHung": false, "majorityOnStalePrimary": { "StartMs": 968230.6262, "EndMs": 968233.5474, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 761, "DurationMs": 2.921199999982491 }, "localOnStalePrimary": { "StartMs": 968230.7209, "EndMs": 968233.6689, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 761, "DurationMs": 2.9480000000912696 }, "linearizableViaReplicaSet": { "StartMs": 973233.605, "EndMs": 974231.3942, "Ok": true, "Error": null, "DocIndex": 0, "Revision": 762, "DurationMs": 997.7891999999993 }, "linearizableSafe": true, "linearizableBounded": true, "conclusive": true, "majorityReturnedRevokedRole": true } ], "verdict": { "Result": "PASS", "LinearizableSupported": true, "DeadlineHeldNoHangs": true, "NoStaleReadAfterAcknowledgedRevoke": true, "DualPrimaryIterations": 3, "ConclusiveDualPrimaryIterations": 3, "RecoveredAutomaticallyAfterEveryFault": true }, "_provenance": { "command": "bash e2e/authority/benchmark/run.sh", "harnessTree": "bcf993d1d1c0d70321c6819cd1c8cfd603a57e4e", "harnessCommit": "3e98d52e5", "date": "2026-09-30", "note": "errorExamples trimmed to message and first driver frames; no other edits" } }