Handle race between index construction and e2e test

[DomAyre]

This will only happen with slow IO, such as during SNP tests on a fileshare

Spun out of #4708 and therefore also contributes to #4190

[ghost]

e2e-fixes@59865 aka 20230106.22 vs main ewma over 20 builds from 59368 to 59857

Click to see table

main

build_id	build_number	Commit latency factor	tpcc_virtual_cft^	tpcc_sgx_cft^	tpcc_sgx_cft_mem	ls_virtual_cft^	ls_sgx_cft^	ls_sgx_cft_mem	pi_ls_virtual_cft^	pi_ls_sgx_cft^	pi_ls_sgx_cft_mem	pi_ls_jwt_virtual_cft^	ls_jwt_virtual_cft^	ls_js_virtual_cft^	pi_ls_jwt_sgx_cft^	pi_ls_jwt_sgx_cft_mem	ls_full_js_virtual_cft^	ls_jwt_sgx_cft^	ls_jwt_sgx_cft_mem	ls_js_jwt_virtual_cft^	ls_js_sgx_cft^	ls_js_sgx_cft_mem	hist_sgx_cft^	ls_full_js_sgx_cft^	ls_full_js_sgx_cft_mem	ls_js_jwt_sgx_cft^	ls_js_jwt_sgx_cft_mem	RB put (/s)^	CHAMP put (/s)^	RB get (/s)^	CHAMP get (/s)^
59368	20221226.2	0.784441	17154.4	6322.91	8.19242e+07	45790.3	15470.9	1.56017e+07	44627.3	15636.5	9.04813e+06	13311	12606.2	4690.73	5825.6	6.68883e+06	3539.47	5860.73	1.48153e+07	3341.07	1570.92	7.73741e+06	25276.6	1282.96	7.21312e+06	1294.42	6.95098e+06	831871	1.17854e+06	8.15105e+06	3.14318e+07
59386	20221227.3	0.794007	17126.1	6374.48	8.1662e+07	43751.6	15754.7	1.53396e+07	44661.3	15660	8.52384e+06	14160.7	14112.5	4739.18	5878.2	6.68883e+06	3641.43	5828.9	1.48153e+07	3338.29	1575.28	7.73741e+06	20740.9	1297.11	7.21312e+06	1308.12	6.95098e+06	835392	1.17908e+06	8.1747e+06	3.07203e+07
59401	20221228.3	0.801802	17307.2	6320.41	8.21863e+07	45743.4	15389.1	1.50774e+07	44649.2	15604.5	8.52384e+06	14164.5	12424.6	4553.25	5831.2	6.68883e+06	3615.21	5869.98	1.50774e+07	3470.03	1568.6	7.73741e+06	22921.1	1287.4	7.73741e+06	1299.41	6.95098e+06	824892	1.17659e+06	8.07992e+06	3.07019e+07
59413	20221229.1	0.804632	17107.7	6319.56	8.24484e+07	45682.8	15454.3	1.50774e+07	47000.2	15626.1	8.78598e+06	13323.8	14122.4	4584.59	5847.8	6.68883e+06	3640.43	5813.09	1.50774e+07	3482.94	1561.62	7.73741e+06	24953.7	1283.06	7.21312e+06	1310	6.95098e+06	841159	1.17652e+06	8.08135e+06	3.11663e+07
59431	20221230.3	0.788723	17208.1	6327.49	8.19242e+07	45920.9	15413.6	1.50774e+07	46720.3	15633.7	8.78598e+06	14190.1	12487.6	4659.78	5850.4	6.68883e+06	3614.84	5868.21	1.45532e+07	3369.98	1572.42	7.73741e+06	24732.4	1288.84	7.73741e+06	1295.77	6.95098e+06	835572	1.17673e+06	8.15595e+06	3.07074e+07
59447	20230102.3	0.799308	17240.6	6247.36	8.24484e+07	43816.5	15456.9	1.53396e+07	46963.2	15605.3	8.78598e+06	14115.7	14164.4	4551.14	5841.3	6.95098e+06	3572.63	5891.02	1.50774e+07	3324.72	1571.91	7.73741e+06	24371.1	1285.66	7.21312e+06	1299.46	6.95098e+06	840360	1.18528e+06	8.1631e+06	3.14728e+07
59462	20230103.3	0.769441	17250.5	6288.77	8.19242e+07	45839	15421.1	1.53396e+07	46662.2	15639.7	9.31027e+06	14009.1	12279.4	4646.76	5847.6	6.68883e+06	3620.99	5822.32	1.50774e+07	3331.48	1572.01	7.73741e+06	27990.6	1285.98	7.21312e+06	1295.7	6.95098e+06	837562	1.17137e+06	8.17082e+06	3.07475e+07
59477	20230103.8	0.778442	17245.4	6320.25	8.21863e+07	42006.4	15465.8	1.50774e+07	46784.3	15598.2	9.31027e+06	13230.7	12362.6	4383.37	5817.1	6.68883e+06	3684.3	5828.86	1.50774e+07	3305.68	1568.13	7.73741e+06	27481.2	1282.18	7.47526e+06	1296.68	6.95098e+06	840264	1.17885e+06	8.15355e+06	3.07933e+07
59503	20230103.14	0.807001	17176.2	6322.06	8.24484e+07	41838.3	15654.4	1.53396e+07	46279.6	15621.7	9.31027e+06	13081.9	12221.1	4529.19	5928.6	6.68883e+06	3530.7	5844.27	1.50774e+07	3399.76	1571.32	7.73741e+06	22258.1	1293.52	7.47526e+06	1297.74	6.95098e+06	835260	1.17714e+06	8.17151e+06	3.01989e+07
59552	20230103.25	0.793474	17049.5	6274.88	8.1662e+07	43539.9	15453	1.50774e+07	45301.4	15583.5	8.52384e+06	13366.6	12521.6	4665.5	5845.5	6.68883e+06	3550.42	5841.39	1.50774e+07	3420.06	1570.79	7.73741e+06	24220.9	1288.84	7.73741e+06	1300.04	6.95098e+06	838063	1.17793e+06	8.04055e+06	3.07351e+07
59578	20230103.30	0.787714	16790.8	6317.76	8.21863e+07	45826.5	15729.5	1.53396e+07	45660	15625.5	9.31027e+06	13368.6	12472.2	4653.88	5808.9	6.68883e+06	3659.56	5816.03	1.50774e+07	3324.89	1573.11	7.73741e+06	23206.6	1289.07	7.21312e+06	1301.95	6.95098e+06	843043	1.18213e+06	8.15092e+06	3.08243e+07
59588	20230104.3	0.791543	17095	6282.06	8.1662e+07	45701.8	15431.4	1.50774e+07	47455.6	15613.9	8.78598e+06	13374.6	12556.1	4670.34	5853.7	6.68883e+06	3701.22	5852	1.48153e+07	3325.31	1571.33	7.73741e+06	24152.6	1283.97	7.73741e+06	1296.71	6.95098e+06	831776	1.18216e+06	8.17317e+06	3.16059e+07
59639	20230104.19	0.793442	17099	6317.48	8.19242e+07	43700.9	15711.9	1.53396e+07	47658.5	15654	8.78598e+06	13311.4	12386.3	4631.54	5888.9	6.95098e+06	3538.75	5834.29	1.50774e+07	3445.45	1565.38	7.73741e+06	27936.5	1289.07	7.73741e+06	1297.73	6.95098e+06	825987	1.18309e+06	8.15566e+06	3.10906e+07
59665	20230105.2	0.79042	17170.6	6325.77	8.21863e+07	43827.9	15559.4	1.53396e+07	47798.4	15682.3	8.78598e+06	13342.3	14174.5	4513.65	5861.5	6.68883e+06	3622.04	5834.47	1.50774e+07	3476.38	1570.57	7.73741e+06	22620.3	1284.7	7.73741e+06	1315.74	6.95098e+06	837979	1.17992e+06	8.17323e+06	3.07162e+07
59678	20230105.6	0.79786	17180.4	6287.89	8.24484e+07	43679.2	15447.8	1.53396e+07	44918.6	15613.9	8.78598e+06	13323.5	12415.2	4590.03	5818	6.68883e+06	3582.73	5834.47	1.50774e+07	3448.03	1571.65	7.73741e+06	27185.3	1284.57	7.73741e+06	1300.41	6.95098e+06	818802	1.15908e+06	8.14269e+06	3.08531e+07
59719	20230105.16	0.816841	17292	6305.07	8.24484e+07	43799.1	15717.1	1.53396e+07	47855.8	15978.5	8.2617e+06	13712.7	12652	4681.08	5810.9	4.59168e+06	3583.49	5800.21	1.50774e+07	3465.9	1572.4	7.73741e+06	23815.7	1293.32	7.47526e+06	1309.53	6.95098e+06	832333	1.17719e+06	8.15582e+06	3.13903e+07
59747	20230105.22	0.794968	17255.8	6282.56	8.21863e+07	43731.7	15541.7	1.50774e+07	48926.1	15864.1	8.2617e+06	12965.6	12297.4	4606.8	5731.4	4.59168e+06	3539.16	5853.15	1.4291e+07	3455.56	1568	7.73741e+06	27253	1270.88	7.47526e+06	1290.39	6.95098e+06	830778	1.17671e+06	8.15459e+06	3.08021e+07
59781	20230106.3	0.766149	17131.4	6304.56	8.21863e+07	45865.6	15485.8	1.53396e+07	47560.6	15996.4	8.2617e+06	13760.5	12520	4556.31	5767.3	4.59168e+06	3545.8	5836.68	1.50774e+07	3482.61	1571.69	7.73741e+06	29148.2	1285.69	7.21312e+06	1301.23	6.95098e+06	830718	1.18073e+06	8.13347e+06	3.12057e+07
59796	20230106.7	0.761166	17319.1	6393.56	8.19242e+07	43805.5	15526.1	1.50774e+07	49223.5	15994	8.2617e+06	13222.1	14194.6	4558.98	5776.3	4.59168e+06	3539.1	5856.54	1.50774e+07	3405.32	1573.51	7.73741e+06	22169.4	1295.62	7.47526e+06	1310.43	6.95098e+06	838036	1.17707e+06	8.15342e+06	3.07106e+07
59857	20230106.21	0.824187	17277.1	5669.72	8.32349e+07	43619.5	15450.5	1.53396e+07	49466.7	15764.6	8.2617e+06	13908.8	12589.5	4670.24	5734.8	4.59168e+06	3547.54	5829.53	1.45532e+07	3478.18	1556.45	7.73741e+06	22136.6	1288.43	7.47526e+06	1286.07	6.95098e+06	834573	1.18175e+06	8.14777e+06	3.08434e+07

e2e-fixes

build_id	build_number	Commit latency factor	tpcc_sgx_cft^	tpcc_sgx_cft_mem	tpcc_virtual_cft^	ls_virtual_cft^	pi_ls_virtual_cft^	ls_sgx_cft^	ls_sgx_cft_mem	pi_ls_sgx_cft^	pi_ls_sgx_cft_mem	ls_jwt_virtual_cft^	pi_ls_jwt_virtual_cft^	ls_jwt_sgx_cft^	ls_jwt_sgx_cft_mem	pi_ls_jwt_sgx_cft^	pi_ls_jwt_sgx_cft_mem	ls_js_virtual_cft^	ls_full_js_virtual_cft^	ls_js_sgx_cft^	ls_js_sgx_cft_mem	ls_js_jwt_virtual_cft^	ls_full_js_sgx_cft^	ls_full_js_sgx_cft_mem	hist_sgx_cft^	ls_js_jwt_sgx_cft^	ls_js_jwt_sgx_cft_mem	RB put (/s)^	CHAMP put (/s)^	RB get (/s)^	CHAMP get (/s)^
59806	20230106.9	0.809892	6360.76	8.1662e+07	16715.4	45655.4	49174.6	15438.5	1.50774e+07	15992.9	8.2617e+06	14306.3	13263.6	5833.59	1.50774e+07	5776.9	4.59168e+06	4534.04	3603.6	1575.49	7.73741e+06	3479.74	1286.49	6.95098e+06	26466.6	1299.61	6.95098e+06	831267	1.17717e+06	8.04298e+06	3.0784e+07
59807	20230106.10	0.785609	6307.64	8.24484e+07	17211.9	45743.8	48690	15486.7	1.53396e+07	15961.4	8.2617e+06	12515.5	13266.5	5834.21	1.50774e+07	5809.5	4.59168e+06	4582.37	3699.01	1572.81	7.73741e+06	3448.52	1285.28	7.21312e+06	22627.4	1309.76	6.95098e+06	827585	1.1766e+06	8.15322e+06	3.07706e+07
59823	20230106.14	0.793771	6289.82	8.19242e+07	17192.7	45758.3	48600.8	15280.9	1.58639e+07	15739.5	7.99955e+06	14062.9	14022.5	5818.78	1.50774e+07	5735	4.59168e+06	4450.31	3657.04	1552.99	7.73741e+06	3442.56	1281.44	7.21312e+06	22943.1	1297.4	6.95098e+06	832742	1.18073e+06	8.09774e+06	3.09258e+07
59865	20230106.22	0.810939	6288.2	8.29727e+07	17222.6	43716.7	46720.6	15488.6	1.50774e+07	15949	8.2617e+06	12605.3	13262.1	5826.57	1.50774e+07	5775.9	4.59168e+06	4368.09	3546.1	1573.9	7.73741e+06	3407.69	1285.8	7.21312e+06	24192.8	1296.34	6.95098e+06	837609	1.17973e+06	8.15462e+06	3.13112e+07