From 42c42bd0c5a8c3438005703bd5eddb7aed526104 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 29 Mar 2020 08:57:18 +0300 Subject: [PATCH 01/20] More aggressive on #448 --- stl/inc/chrono | 56 +++++++++++++++++++++++++++++++-- tests/tr1/tests/chrono/test.cpp | 30 ++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 4704c6c9241..c4875f1ad18 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -8,6 +8,7 @@ #define _CHRONO_ #include #if _STL_COMPILER_PREPROCESSOR +#include #include #include #include @@ -609,13 +610,62 @@ namespace chrono { static constexpr bool is_steady = true; _NODISCARD static time_point now() noexcept { // get current time +#if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) + // Implement atomics avoiding header dependency + static volatile long long _Freq_cached = -1; + static volatile long long _Quot_cached = -1; + static volatile long long _Rem_cached = -1; + long long _Freq = _Atomic_load_ll_relaxed(&_Freq_cached); + if (_Freq == -1) { + _Freq = _Query_perf_frequency(); + _Atomic_store_ll_relaxed(&_Freq_cached, _Freq, -1); + } + long long _Quot = _Atomic_load_ll_relaxed(&_Quot_cached); + long long _Rem = _Atomic_load_ll_relaxed(&_Rem_cached); + if (_Quot == -1 || _Rem == -1) { + static_assert(period::num == 1, "This assumes period::num == 1."); + _Quot = period::den / _Freq; + _Rem = period::den % _Freq; + _Atomic_store_ll_relaxed(&_Quot_cached, _Quot, -1); + _Atomic_store_ll_relaxed(&_Rem_cached, _Rem, -1); + } +#else // ^^^ known hardware && !defined(_M_CEE_PURE) / unknown hardware || defined(_M_CEE_PURE) vvv const long long _Freq = _Query_perf_frequency(); // doesn't change after system boot - const long long _Ctr = _Query_perf_counter(); static_assert(period::num == 1, "This assumes period::num == 1."); - const long long _Whole = (_Ctr / _Freq) * period::den; - const long long _Part = (_Ctr % _Freq) * period::den / _Freq; + const long long _Quot = period::den / _Freq; + const long long _Rem = period::den % _Freq; +#endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) + const long long _Ctr = _Query_perf_counter(); + const long long _Whole = _Quot * _Ctr; + const long long _Part = _Rem * _Ctr / _Freq; return time_point(duration(_Whole + _Part)); } + +#if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) + static long long _Atomic_load_ll_relaxed(volatile long long* _Mem) { + // Copy from _Atomic_storage<_Ty, 8>::load +#if defined(_M_IX86) || defined(_M_ARM64) + return __iso_volatile_load64(_Mem); +#elif defined(_M_X64) + return *_Mem; +#else // _M_ARM + return __ldrexd(_Mem); +#endif // hardware + } + + static void _Atomic_store_ll_relaxed(volatile long long* _Mem, long long _Value, long long _Assumed_mem) { + (void) _Assumed_mem; // maybe unused. Useful for x86 to avoid loop + // Copy from _Atomic_storage<_Ty, 8>::store +#if defined(_M_IX86) + _InterlockedCompareExchange64(_Mem, _Value, _Assumed_mem); +#elif defined(_M_X64) + *_Mem = _Value; +#else // defined(_M_ARM) || defined(_M_ARM64) + __iso_volatile_store64(_Mem, _Value); +#endif // hardware + } +#endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) + }; using high_resolution_clock = steady_clock; diff --git a/tests/tr1/tests/chrono/test.cpp b/tests/tr1/tests/chrono/test.cpp index a87b38a25cf..c9abc1538f7 100644 --- a/tests/tr1/tests/chrono/test.cpp +++ b/tests/tr1/tests/chrono/test.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace { @@ -297,6 +298,34 @@ namespace { } } + template + void test_sleep() { + auto t1 = ClockToTestAgainst::now(); + auto t2 = ClockBeingTested::now(); + auto t3 = ClockToTestAgainst::now(); + + auto sleep_time = STD chrono::milliseconds(2); + STD this_thread::sleep_for(sleep_time); + + auto d4 = STD chrono::duration_cast(sleep_time); + + auto d3 = STD chrono::duration_cast(ClockToTestAgainst::now() - t3); + auto d2 = STD chrono::duration_cast(ClockBeingTested::now() - t2); + auto d1 = STD chrono::duration_cast(ClockToTestAgainst::now() - t1); + + CHECK_INT(d4.count() <= d3.count(), true); + CHECK_INT(d3.count() <= d2.count(), true); + CHECK_INT(d2.count() <= d1.count(), true); + } + + void t_sleep() { + test_sleep(); + + test_sleep(); + } + } // anonymous namespace void test_main() { @@ -306,4 +335,5 @@ void test_main() { t_duration(); t_clocks(); t_time_point(); + t_sleep(); } From baf428c16a0fde903b98fae3bc1926e854733075 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 29 Mar 2020 09:24:44 +0300 Subject: [PATCH 02/20] clang format --- stl/inc/chrono | 1 - tests/tr1/tests/chrono/test.cpp | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index c4875f1ad18..d5df5a17ff6 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -665,7 +665,6 @@ namespace chrono { #endif // hardware } #endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) - }; using high_resolution_clock = steady_clock; diff --git a/tests/tr1/tests/chrono/test.cpp b/tests/tr1/tests/chrono/test.cpp index c9abc1538f7..9c0e266db5d 100644 --- a/tests/tr1/tests/chrono/test.cpp +++ b/tests/tr1/tests/chrono/test.cpp @@ -7,9 +7,9 @@ #include "tdefs.h" #include #include +#include #include #include -#include namespace { @@ -303,7 +303,7 @@ namespace { auto t1 = ClockToTestAgainst::now(); auto t2 = ClockBeingTested::now(); auto t3 = ClockToTestAgainst::now(); - + auto sleep_time = STD chrono::milliseconds(2); STD this_thread::sleep_for(sleep_time); From 315299ce2f2f9872486746fc6fd8ba28c40d9937 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 29 Mar 2020 12:03:44 +0300 Subject: [PATCH 03/20] Group variables to reduce branching --- stl/inc/chrono | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index d5df5a17ff6..34c62b07fb6 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -612,22 +612,22 @@ namespace chrono { _NODISCARD static time_point now() noexcept { // get current time #if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) // Implement atomics avoiding header dependency - static volatile long long _Freq_cached = -1; - static volatile long long _Quot_cached = -1; - static volatile long long _Rem_cached = -1; - long long _Freq = _Atomic_load_ll_relaxed(&_Freq_cached); - if (_Freq == -1) { - _Freq = _Query_perf_frequency(); - _Atomic_store_ll_relaxed(&_Freq_cached, _Freq, -1); - } - long long _Quot = _Atomic_load_ll_relaxed(&_Quot_cached); - long long _Rem = _Atomic_load_ll_relaxed(&_Rem_cached); - if (_Quot == -1 || _Rem == -1) { + static struct { + volatile long long _Freq = -1; + volatile long long _Quot = -1; + volatile long long _Rem = -1; + } _Cached; + long long _Freq = _Atomic_load_ll_relaxed(&_Cached._Freq); + long long _Quot = _Atomic_load_ll_relaxed(&_Cached._Quot); + long long _Rem = _Atomic_load_ll_relaxed(&_Cached._Rem); + if ((_Freq | _Quot | _Rem) < 0) { static_assert(period::num == 1, "This assumes period::num == 1."); + _Freq = _Query_perf_frequency(); _Quot = period::den / _Freq; _Rem = period::den % _Freq; - _Atomic_store_ll_relaxed(&_Quot_cached, _Quot, -1); - _Atomic_store_ll_relaxed(&_Rem_cached, _Rem, -1); + _Atomic_store_ll_relaxed(&_Cached._Freq, _Freq, -1); + _Atomic_store_ll_relaxed(&_Cached._Quot, _Quot, -1); + _Atomic_store_ll_relaxed(&_Cached._Rem, _Rem, -1); } #else // ^^^ known hardware && !defined(_M_CEE_PURE) / unknown hardware || defined(_M_CEE_PURE) vvv const long long _Freq = _Query_perf_frequency(); // doesn't change after system boot From 404a6feab6f94c35ecda65590d8482fd3166c94b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 29 Mar 2020 15:10:49 +0300 Subject: [PATCH 04/20] unsigned divides faster --- stl/inc/chrono | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 34c62b07fb6..6e6b57240bd 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -637,7 +637,9 @@ namespace chrono { #endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) const long long _Ctr = _Query_perf_counter(); const long long _Whole = _Quot * _Ctr; - const long long _Part = _Rem * _Ctr / _Freq; + // cast to unsigned for faster division + const long long _Part = static_cast( + static_cast(_Rem * _Ctr) / static_cast(_Freq)); return time_point(duration(_Whole + _Part)); } From 004426d64ff0168fc87b22cd7bb285aea272d1aa Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 29 Mar 2020 15:25:29 +0300 Subject: [PATCH 05/20] Revert "unsigned divides faster" Actually there's no quarantee in docs that QPC is non-negative --- stl/inc/chrono | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 6e6b57240bd..34c62b07fb6 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -637,9 +637,7 @@ namespace chrono { #endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) const long long _Ctr = _Query_perf_counter(); const long long _Whole = _Quot * _Ctr; - // cast to unsigned for faster division - const long long _Part = static_cast( - static_cast(_Rem * _Ctr) / static_cast(_Freq)); + const long long _Part = _Rem * _Ctr / _Freq; return time_point(duration(_Whole + _Part)); } From 444c82f5d62c871b9fefd6fab79dd7024ba1bb10 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 29 Mar 2020 21:08:00 +0300 Subject: [PATCH 06/20] Clear comparison against uninitialized performance difference cannot be noticed --- stl/inc/chrono | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 34c62b07fb6..6472db96262 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -620,7 +620,7 @@ namespace chrono { long long _Freq = _Atomic_load_ll_relaxed(&_Cached._Freq); long long _Quot = _Atomic_load_ll_relaxed(&_Cached._Quot); long long _Rem = _Atomic_load_ll_relaxed(&_Cached._Rem); - if ((_Freq | _Quot | _Rem) < 0) { + if (_Freq == -1 || _Quot == -1 || _Rem == -1) { static_assert(period::num == 1, "This assumes period::num == 1."); _Freq = _Query_perf_frequency(); _Quot = period::den / _Freq; From d7c8a4c10e266c5285c8fac12a4f4c1a03d4f693 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 2 Apr 2020 12:25:01 +0300 Subject: [PATCH 07/20] Rebase counter and do one division --- stl/inc/chrono | 60 +++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 6472db96262..bea25ec0a67 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -613,32 +613,42 @@ namespace chrono { #if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) // Implement atomics avoiding header dependency static struct { - volatile long long _Freq = -1; - volatile long long _Quot = -1; - volatile long long _Rem = -1; + volatile long long _Freq = LLONG_MAX; + volatile long long _Ctr_base = LLONG_MAX; + volatile long long _Result_base = LLONG_MAX; } _Cached; - long long _Freq = _Atomic_load_ll_relaxed(&_Cached._Freq); - long long _Quot = _Atomic_load_ll_relaxed(&_Cached._Quot); - long long _Rem = _Atomic_load_ll_relaxed(&_Cached._Rem); - if (_Freq == -1 || _Quot == -1 || _Rem == -1) { + long long _Freq; + long long _Ctr_base; + long long _Result_base; + for (;;) { + _Freq = _Atomic_load_ll_relaxed(&_Cached._Freq); + _Ctr_base = _Atomic_load_ll_relaxed(&_Cached._Ctr_base); + _Result_base = _Atomic_load_ll_relaxed(&_Cached._Result_base); + if (_Freq != LLONG_MAX && _Ctr_base != LLONG_MAX && _Result_base != LLONG_MAX) { + break; + } static_assert(period::num == 1, "This assumes period::num == 1."); - _Freq = _Query_perf_frequency(); - _Quot = period::den / _Freq; - _Rem = period::den % _Freq; - _Atomic_store_ll_relaxed(&_Cached._Freq, _Freq, -1); - _Atomic_store_ll_relaxed(&_Cached._Quot, _Quot, -1); - _Atomic_store_ll_relaxed(&_Cached._Rem, _Rem, -1); + _Freq = _Query_perf_frequency(); + _Ctr_base = _Query_perf_counter(); + const long long _Whole = (_Ctr_base / _Freq) * period::den; + const long long _Part = (_Ctr_base % _Freq) * period::den / _Freq; + _Result_base = _Whole + _Part; + if (_Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Freq, _Freq, LLONG_MAX) && + _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Ctr_base, _Ctr_base, LLONG_MAX) && + _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Result_base, _Result_base, LLONG_MAX)) { + return time_point(duration(_Result_base)); + } } + const long long _Ctr = _Query_perf_counter(); + return time_point(duration((_Ctr - _Ctr_base) * period::den / _Freq + _Result_base)); #else // ^^^ known hardware && !defined(_M_CEE_PURE) / unknown hardware || defined(_M_CEE_PURE) vvv const long long _Freq = _Query_perf_frequency(); // doesn't change after system boot + const long long _Ctr = _Query_perf_counter(); static_assert(period::num == 1, "This assumes period::num == 1."); - const long long _Quot = period::den / _Freq; - const long long _Rem = period::den % _Freq; -#endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) - const long long _Ctr = _Query_perf_counter(); - const long long _Whole = _Quot * _Ctr; - const long long _Part = _Rem * _Ctr / _Freq; + const long long _Whole = (_Ctr / _Freq) * period::den; + const long long _Part = (_Ctr % _Freq) * period::den / _Freq; return time_point(duration(_Whole + _Part)); +#endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) } #if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) @@ -653,16 +663,10 @@ namespace chrono { #endif // hardware } - static void _Atomic_store_ll_relaxed(volatile long long* _Mem, long long _Value, long long _Assumed_mem) { - (void) _Assumed_mem; // maybe unused. Useful for x86 to avoid loop + static bool _Atomic_compare_exchange_strong_ll_cst_seq( + volatile long long* _Mem, long long _Value, long long _Comparand) { // Copy from _Atomic_storage<_Ty, 8>::store -#if defined(_M_IX86) - _InterlockedCompareExchange64(_Mem, _Value, _Assumed_mem); -#elif defined(_M_X64) - *_Mem = _Value; -#else // defined(_M_ARM) || defined(_M_ARM64) - __iso_volatile_store64(_Mem, _Value); -#endif // hardware + return _InterlockedCompareExchange64(_Mem, _Value, _Comparand) == _Comparand; } #endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) }; From aeecbcd91cb9969d620670d5834c08255cead610 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 2 Apr 2020 12:29:12 +0300 Subject: [PATCH 08/20] Undo test, it is not correct test --- tests/tr1/tests/chrono/test.cpp | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/tests/tr1/tests/chrono/test.cpp b/tests/tr1/tests/chrono/test.cpp index 9c0e266db5d..a87b38a25cf 100644 --- a/tests/tr1/tests/chrono/test.cpp +++ b/tests/tr1/tests/chrono/test.cpp @@ -7,7 +7,6 @@ #include "tdefs.h" #include #include -#include #include #include @@ -298,34 +297,6 @@ namespace { } } - template - void test_sleep() { - auto t1 = ClockToTestAgainst::now(); - auto t2 = ClockBeingTested::now(); - auto t3 = ClockToTestAgainst::now(); - - auto sleep_time = STD chrono::milliseconds(2); - STD this_thread::sleep_for(sleep_time); - - auto d4 = STD chrono::duration_cast(sleep_time); - - auto d3 = STD chrono::duration_cast(ClockToTestAgainst::now() - t3); - auto d2 = STD chrono::duration_cast(ClockBeingTested::now() - t2); - auto d1 = STD chrono::duration_cast(ClockToTestAgainst::now() - t1); - - CHECK_INT(d4.count() <= d3.count(), true); - CHECK_INT(d3.count() <= d2.count(), true); - CHECK_INT(d2.count() <= d1.count(), true); - } - - void t_sleep() { - test_sleep(); - - test_sleep(); - } - } // anonymous namespace void test_main() { @@ -335,5 +306,4 @@ void test_main() { t_duration(); t_clocks(); t_time_point(); - t_sleep(); } From 7e57c0541d1148a9b0da83d0faac838ec77921f6 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 2 Apr 2020 12:37:35 +0300 Subject: [PATCH 09/20] cland format --- stl/inc/chrono | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index bea25ec0a67..44d1da30b7a 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -633,9 +633,9 @@ namespace chrono { const long long _Whole = (_Ctr_base / _Freq) * period::den; const long long _Part = (_Ctr_base % _Freq) * period::den / _Freq; _Result_base = _Whole + _Part; - if (_Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Freq, _Freq, LLONG_MAX) && - _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Ctr_base, _Ctr_base, LLONG_MAX) && - _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Result_base, _Result_base, LLONG_MAX)) { + if (_Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Freq, _Freq, LLONG_MAX) + && _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Ctr_base, _Ctr_base, LLONG_MAX) + && _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Result_base, _Result_base, LLONG_MAX)) { return time_point(duration(_Result_base)); } } From a255604087d3a2354d3700d191eb713469204854 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 2 Apr 2020 13:18:26 +0300 Subject: [PATCH 10/20] avoid loop --- stl/inc/chrono | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 44d1da30b7a..cd4b34dfa8c 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -617,30 +617,28 @@ namespace chrono { volatile long long _Ctr_base = LLONG_MAX; volatile long long _Result_base = LLONG_MAX; } _Cached; - long long _Freq; - long long _Ctr_base; - long long _Result_base; - for (;;) { - _Freq = _Atomic_load_ll_relaxed(&_Cached._Freq); - _Ctr_base = _Atomic_load_ll_relaxed(&_Cached._Ctr_base); - _Result_base = _Atomic_load_ll_relaxed(&_Cached._Result_base); - if (_Freq != LLONG_MAX && _Ctr_base != LLONG_MAX && _Result_base != LLONG_MAX) { - break; - } - static_assert(period::num == 1, "This assumes period::num == 1."); - _Freq = _Query_perf_frequency(); - _Ctr_base = _Query_perf_counter(); - const long long _Whole = (_Ctr_base / _Freq) * period::den; - const long long _Part = (_Ctr_base % _Freq) * period::den / _Freq; - _Result_base = _Whole + _Part; - if (_Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Freq, _Freq, LLONG_MAX) - && _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Ctr_base, _Ctr_base, LLONG_MAX) - && _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Result_base, _Result_base, LLONG_MAX)) { - return time_point(duration(_Result_base)); - } + long long _Freq = _Atomic_load_ll_relaxed(&_Cached._Freq); + const long long _Ctr_base = _Atomic_load_ll_relaxed(&_Cached._Ctr_base); + const long long _Result_base = _Atomic_load_ll_relaxed(&_Cached._Result_base); + if (_Freq != LLONG_MAX && _Ctr_base != LLONG_MAX && _Result_base != LLONG_MAX) { + // Fast path + const long long _Ctr = _Query_perf_counter(); + return time_point(duration((_Ctr - _Ctr_base) * period::den / _Freq + _Result_base)); } + // Caclulate with two divisions to prevent overflow + _Freq = _Query_perf_frequency(); const long long _Ctr = _Query_perf_counter(); - return time_point(duration((_Ctr - _Ctr_base) * period::den / _Freq + _Result_base)); + static_assert(period::num == 1, "This assumes period::num == 1."); + const long long _Whole = (_Ctr / _Freq) * period::den; + const long long _Part = (_Ctr % _Freq) * period::den / _Freq; + const long long _Result = _Whole + _Part; + if (_Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Freq, _Freq, LLONG_MAX)) { + // This is the first result, save current result as base for fast path + _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Ctr_base, _Ctr, LLONG_MAX); + _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Result_base, _Result, LLONG_MAX); + } + // if _Result is not saved as first, it is still compatible with fast result + return time_point(duration(_Result)); #else // ^^^ known hardware && !defined(_M_CEE_PURE) / unknown hardware || defined(_M_CEE_PURE) vvv const long long _Freq = _Query_perf_frequency(); // doesn't change after system boot const long long _Ctr = _Query_perf_counter(); From fc19466ae5fba0a261ba8d421bd151b103f00a3c Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 2 Apr 2020 15:08:53 +0300 Subject: [PATCH 11/20] trim whitespace --- stl/inc/chrono | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index cd4b34dfa8c..a716fe0b822 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -638,7 +638,7 @@ namespace chrono { _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Result_base, _Result, LLONG_MAX); } // if _Result is not saved as first, it is still compatible with fast result - return time_point(duration(_Result)); + return time_point(duration(_Result)); #else // ^^^ known hardware && !defined(_M_CEE_PURE) / unknown hardware || defined(_M_CEE_PURE) vvv const long long _Freq = _Query_perf_frequency(); // doesn't change after system boot const long long _Ctr = _Query_perf_counter(); From 27db1baaf4f545c68240a0d00df6b5efd37fce11 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 11 Apr 2020 07:12:00 +0300 Subject: [PATCH 12/20] After merging with #658 , avoid explanation repetition --- stl/inc/chrono | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 5d0052327da..5849165ea31 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -609,6 +609,18 @@ namespace chrono { using time_point = chrono::time_point; static constexpr bool is_steady = true; + static long long _Scale_large_counter(const long long _Ctr, const long long _Freq) { + static_assert(period::num == 1, "This assumes period::num == 1."); + // Instead of just having "(_Ctr * period::den) / _Freq", + // the algorithm below prevents overflow when _Ctr is sufficiently large. + // It assumes that _Freq * period::den does not overflow, which is currently true for nano period. + // It is not realistic for _Ctr to accumulate to large values from zero with this assumption, + // but the initial value of _Ctr could be large. + const long long _Whole = (_Ctr / _Freq) * period::den; + const long long _Part = (_Ctr % _Freq) * period::den / _Freq; + return _Whole + _Part; + } + _NODISCARD static time_point now() noexcept { // get current time #if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) // Implement atomics avoiding header dependency @@ -628,10 +640,7 @@ namespace chrono { // Caclulate with two divisions to prevent overflow _Freq = _Query_perf_frequency(); const long long _Ctr = _Query_perf_counter(); - static_assert(period::num == 1, "This assumes period::num == 1."); - const long long _Whole = (_Ctr / _Freq) * period::den; - const long long _Part = (_Ctr % _Freq) * period::den / _Freq; - const long long _Result = _Whole + _Part; + const long long _Result = _Scale_large_counter(_Ctr, _Freq); if (_Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Freq, _Freq, LLONG_MAX)) { // This is the first result, save current result as base for fast path _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Ctr_base, _Ctr, LLONG_MAX); @@ -642,15 +651,7 @@ namespace chrono { #else // ^^^ known hardware && !defined(_M_CEE_PURE) / unknown hardware || defined(_M_CEE_PURE) vvv const long long _Freq = _Query_perf_frequency(); // doesn't change after system boot const long long _Ctr = _Query_perf_counter(); - static_assert(period::num == 1, "This assumes period::num == 1."); - // Instead of just having "(_Ctr * period::den) / _Freq", - // the algorithm below prevents overflow when _Ctr is sufficiently large. - // It assumes that _Freq * period::den does not overflow, which is currently true for nano period. - // It is not realistic for _Ctr to accumulate to large values from zero with this assumption, - // but the initial value of _Ctr could be large. - const long long _Whole = (_Ctr / _Freq) * period::den; - const long long _Part = (_Ctr % _Freq) * period::den / _Freq; - return time_point(duration(_Whole + _Part)); + return time_point(duration(_Scale_large_counter(_Ctr, _Freq))); #endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) } From bf63ab6d2dc3bf4bb2496800639c4cd888e6404e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 11 Apr 2020 07:19:19 +0300 Subject: [PATCH 13/20] clang format --- stl/inc/chrono | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 5849165ea31..a4cbafd977e 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -638,8 +638,8 @@ namespace chrono { return time_point(duration((_Ctr - _Ctr_base) * period::den / _Freq + _Result_base)); } // Caclulate with two divisions to prevent overflow - _Freq = _Query_perf_frequency(); - const long long _Ctr = _Query_perf_counter(); + _Freq = _Query_perf_frequency(); + const long long _Ctr = _Query_perf_counter(); const long long _Result = _Scale_large_counter(_Ctr, _Freq); if (_Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Freq, _Freq, LLONG_MAX)) { // This is the first result, save current result as base for fast path From e85274a8c8fe39f3cddb0d9e87040b485733e972 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 11 Apr 2020 07:21:21 +0300 Subject: [PATCH 14/20] more const --- stl/inc/chrono | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index a4cbafd977e..241127cb7fe 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -629,16 +629,16 @@ namespace chrono { volatile long long _Ctr_base = LLONG_MAX; volatile long long _Result_base = LLONG_MAX; } _Cached; - long long _Freq = _Atomic_load_ll_relaxed(&_Cached._Freq); - const long long _Ctr_base = _Atomic_load_ll_relaxed(&_Cached._Ctr_base); - const long long _Result_base = _Atomic_load_ll_relaxed(&_Cached._Result_base); - if (_Freq != LLONG_MAX && _Ctr_base != LLONG_MAX && _Result_base != LLONG_MAX) { + const long long _Freq_from_cache = _Atomic_load_ll_relaxed(&_Cached._Freq); + const long long _Ctr_base = _Atomic_load_ll_relaxed(&_Cached._Ctr_base); + const long long _Result_base = _Atomic_load_ll_relaxed(&_Cached._Result_base); + if (_Freq_from_cache != LLONG_MAX && _Ctr_base != LLONG_MAX && _Result_base != LLONG_MAX) { // Fast path const long long _Ctr = _Query_perf_counter(); - return time_point(duration((_Ctr - _Ctr_base) * period::den / _Freq + _Result_base)); + return time_point(duration((_Ctr - _Ctr_base) * period::den / _Freq_from_cache + _Result_base)); } // Caclulate with two divisions to prevent overflow - _Freq = _Query_perf_frequency(); + const long long _Freq = _Query_perf_frequency(); const long long _Ctr = _Query_perf_counter(); const long long _Result = _Scale_large_counter(_Ctr, _Freq); if (_Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Freq, _Freq, LLONG_MAX)) { From 178b9bfa638de794f81863104202af0fc0e3bf0c Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 11 May 2020 19:59:48 -0700 Subject: [PATCH 15/20] Code review feedback, part 1. --- stl/inc/chrono | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 241127cb7fe..4f90cd4d2ec 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -609,7 +609,7 @@ namespace chrono { using time_point = chrono::time_point; static constexpr bool is_steady = true; - static long long _Scale_large_counter(const long long _Ctr, const long long _Freq) { + _NODISCARD static long long _Scale_large_counter(const long long _Ctr, const long long _Freq) noexcept { static_assert(period::num == 1, "This assumes period::num == 1."); // Instead of just having "(_Ctr * period::den) / _Freq", // the algorithm below prevents overflow when _Ctr is sufficiently large. @@ -635,16 +635,16 @@ namespace chrono { if (_Freq_from_cache != LLONG_MAX && _Ctr_base != LLONG_MAX && _Result_base != LLONG_MAX) { // Fast path const long long _Ctr = _Query_perf_counter(); - return time_point(duration((_Ctr - _Ctr_base) * period::den / _Freq_from_cache + _Result_base)); + return time_point(duration(_Result_base + (_Ctr - _Ctr_base) * period::den / _Freq_from_cache)); } - // Caclulate with two divisions to prevent overflow + // Calculate with two divisions to prevent overflow const long long _Freq = _Query_perf_frequency(); const long long _Ctr = _Query_perf_counter(); const long long _Result = _Scale_large_counter(_Ctr, _Freq); - if (_Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Freq, _Freq, LLONG_MAX)) { + if (_Atomic_compare_exchange_strong_ll_seq_cst(&_Cached._Freq, _Freq, LLONG_MAX)) { // This is the first result, save current result as base for fast path - _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Ctr_base, _Ctr, LLONG_MAX); - _Atomic_compare_exchange_strong_ll_cst_seq(&_Cached._Result_base, _Result, LLONG_MAX); + _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached._Ctr_base, _Ctr, LLONG_MAX); + _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached._Result_base, _Result, LLONG_MAX); } // if _Result is not saved as first, it is still compatible with fast result return time_point(duration(_Result)); @@ -656,7 +656,7 @@ namespace chrono { } #if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) - static long long _Atomic_load_ll_relaxed(volatile long long* _Mem) { + _NODISCARD static long long _Atomic_load_ll_relaxed(volatile long long* _Mem) noexcept { // Copy from _Atomic_storage<_Ty, 8>::load #if defined(_M_IX86) || defined(_M_ARM64) return __iso_volatile_load64(_Mem); @@ -667,8 +667,8 @@ namespace chrono { #endif // hardware } - static bool _Atomic_compare_exchange_strong_ll_cst_seq( - volatile long long* _Mem, long long _Value, long long _Comparand) { + static bool _Atomic_compare_exchange_strong_ll_seq_cst( + volatile long long* _Mem, long long _Value, long long _Comparand) noexcept { // Copy from _Atomic_storage<_Ty, 8>::store return _InterlockedCompareExchange64(_Mem, _Value, _Comparand) == _Comparand; } From 9d686ec29ab744764e92f861fa97d5219188f640 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 11 May 2020 20:02:53 -0700 Subject: [PATCH 16/20] Code review feedback, part 2: access control. --- stl/inc/chrono | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 4f90cd4d2ec..0bdf45c7d36 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -609,18 +609,6 @@ namespace chrono { using time_point = chrono::time_point; static constexpr bool is_steady = true; - _NODISCARD static long long _Scale_large_counter(const long long _Ctr, const long long _Freq) noexcept { - static_assert(period::num == 1, "This assumes period::num == 1."); - // Instead of just having "(_Ctr * period::den) / _Freq", - // the algorithm below prevents overflow when _Ctr is sufficiently large. - // It assumes that _Freq * period::den does not overflow, which is currently true for nano period. - // It is not realistic for _Ctr to accumulate to large values from zero with this assumption, - // but the initial value of _Ctr could be large. - const long long _Whole = (_Ctr / _Freq) * period::den; - const long long _Part = (_Ctr % _Freq) * period::den / _Freq; - return _Whole + _Part; - } - _NODISCARD static time_point now() noexcept { // get current time #if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) // Implement atomics avoiding header dependency @@ -655,6 +643,19 @@ namespace chrono { #endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) } + private: + _NODISCARD static long long _Scale_large_counter(const long long _Ctr, const long long _Freq) noexcept { + static_assert(period::num == 1, "This assumes period::num == 1."); + // Instead of just having "(_Ctr * period::den) / _Freq", + // the algorithm below prevents overflow when _Ctr is sufficiently large. + // It assumes that _Freq * period::den does not overflow, which is currently true for nano period. + // It is not realistic for _Ctr to accumulate to large values from zero with this assumption, + // but the initial value of _Ctr could be large. + const long long _Whole = (_Ctr / _Freq) * period::den; + const long long _Part = (_Ctr % _Freq) * period::den / _Freq; + return _Whole + _Part; + } + #if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) _NODISCARD static long long _Atomic_load_ll_relaxed(volatile long long* _Mem) noexcept { // Copy from _Atomic_storage<_Ty, 8>::load From 2347e1014fc92016e9d31b8195ba83a9d97c039f Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 12 May 2020 06:47:58 +0300 Subject: [PATCH 17/20] Code review feedback, part 3: avoid magic static even with /Od --- stl/inc/chrono | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 0bdf45c7d36..39491857994 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -612,14 +612,13 @@ namespace chrono { _NODISCARD static time_point now() noexcept { // get current time #if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) // Implement atomics avoiding header dependency - static struct { - volatile long long _Freq = LLONG_MAX; - volatile long long _Ctr_base = LLONG_MAX; - volatile long long _Result_base = LLONG_MAX; - } _Cached; - const long long _Freq_from_cache = _Atomic_load_ll_relaxed(&_Cached._Freq); - const long long _Ctr_base = _Atomic_load_ll_relaxed(&_Cached._Ctr_base); - const long long _Result_base = _Atomic_load_ll_relaxed(&_Cached._Result_base); + static volatile long long _Cached_Freq = LLONG_MAX; + static volatile long long _Cached_Ctr_base = LLONG_MAX; + static volatile long long _Cached_Result_base = LLONG_MAX; + + const long long _Freq_from_cache = _Atomic_load_ll_relaxed(&_Cached_Freq); + const long long _Ctr_base = _Atomic_load_ll_relaxed(&_Cached_Ctr_base); + const long long _Result_base = _Atomic_load_ll_relaxed(&_Cached_Result_base); if (_Freq_from_cache != LLONG_MAX && _Ctr_base != LLONG_MAX && _Result_base != LLONG_MAX) { // Fast path const long long _Ctr = _Query_perf_counter(); @@ -629,10 +628,10 @@ namespace chrono { const long long _Freq = _Query_perf_frequency(); const long long _Ctr = _Query_perf_counter(); const long long _Result = _Scale_large_counter(_Ctr, _Freq); - if (_Atomic_compare_exchange_strong_ll_seq_cst(&_Cached._Freq, _Freq, LLONG_MAX)) { + if (_Atomic_compare_exchange_strong_ll_seq_cst(&_Cached_Freq, _Freq, LLONG_MAX)) { // This is the first result, save current result as base for fast path - _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached._Ctr_base, _Ctr, LLONG_MAX); - _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached._Result_base, _Result, LLONG_MAX); + _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached_Ctr_base, _Ctr, LLONG_MAX); + _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached_Result_base, _Result, LLONG_MAX); } // if _Result is not saved as first, it is still compatible with fast result return time_point(duration(_Result)); From 156783fca9ec3b43fe7b604bbf78c021ca0394d4 Mon Sep 17 00:00:00 2001 From: Casey Carter Date: Tue, 12 May 2020 12:53:48 -0700 Subject: [PATCH 18/20] Relocate two _Atomic_meow function templates to simply for physical proximity, so we'll maintain them together with the rest of the atomic machinery. --- stl/inc/chrono | 20 +------------------- stl/inc/xatomic.h | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 39491857994..4e6ece5a7ca 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -13,6 +13,7 @@ #include #include #include +#include #include #pragma pack(push, _CRT_PACKING) @@ -654,25 +655,6 @@ namespace chrono { const long long _Part = (_Ctr % _Freq) * period::den / _Freq; return _Whole + _Part; } - -#if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) - _NODISCARD static long long _Atomic_load_ll_relaxed(volatile long long* _Mem) noexcept { - // Copy from _Atomic_storage<_Ty, 8>::load -#if defined(_M_IX86) || defined(_M_ARM64) - return __iso_volatile_load64(_Mem); -#elif defined(_M_X64) - return *_Mem; -#else // _M_ARM - return __ldrexd(_Mem); -#endif // hardware - } - - static bool _Atomic_compare_exchange_strong_ll_seq_cst( - volatile long long* _Mem, long long _Value, long long _Comparand) noexcept { - // Copy from _Atomic_storage<_Ty, 8>::store - return _InterlockedCompareExchange64(_Mem, _Value, _Comparand) == _Comparand; - } -#endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) }; using high_resolution_clock = steady_clock; diff --git a/stl/inc/xatomic.h b/stl/inc/xatomic.h index 6af55a96696..970d2fd4340 100644 --- a/stl/inc/xatomic.h +++ b/stl/inc/xatomic.h @@ -104,6 +104,27 @@ _NODISCARD volatile _Integral* _Atomic_address_as(_Ty& _Source) noexcept { return &reinterpret_cast(_Source); } +// FUNCTION TEMPLATE _Atomic_load_ll_relaxed +#if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) +_NODISCARD inline long long _Atomic_load_ll_relaxed(volatile long long* _Mem) noexcept { + // Copy from _Atomic_storage<_Ty, 8>::load +#if defined(_M_IX86) || defined(_M_ARM64) + return __iso_volatile_load64(_Mem); +#elif defined(_M_X64) + return *_Mem; +#else // _M_ARM + return __ldrexd(_Mem); +#endif // hardware +} + +// FUNCTION TEMPLATE _Atomic_compare_exchange_strong_ll_seq_cst +inline bool _Atomic_compare_exchange_strong_ll_seq_cst( + volatile long long* _Mem, long long _Value, long long _Comparand) noexcept { + // Copy from _Atomic_storage<_Ty, 8>::store + return _InterlockedCompareExchange64(_Mem, _Value, _Comparand) == _Comparand; +} +#endif // (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) + _STD_END #pragma pop_macro("new") From f72c96c7757cbebc79f6880a84617cf9363d3428 Mon Sep 17 00:00:00 2001 From: Casey Carter Date: Tue, 12 May 2020 12:56:22 -0700 Subject: [PATCH 19/20] Remove now-unused include in --- stl/inc/chrono | 1 - 1 file changed, 1 deletion(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 4e6ece5a7ca..071390fd3b0 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -8,7 +8,6 @@ #define _CHRONO_ #include #if _STL_COMPILER_PREPROCESSOR -#include #include #include #include From 4583b844120641adc179121f74ffb20aceed7d7d Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 12 May 2020 14:27:28 -0700 Subject: [PATCH 20/20] Use _Ugly_snake_case. --- stl/inc/chrono | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/stl/inc/chrono b/stl/inc/chrono index 071390fd3b0..1eb85bd4beb 100644 --- a/stl/inc/chrono +++ b/stl/inc/chrono @@ -612,13 +612,13 @@ namespace chrono { _NODISCARD static time_point now() noexcept { // get current time #if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) // Implement atomics avoiding header dependency - static volatile long long _Cached_Freq = LLONG_MAX; - static volatile long long _Cached_Ctr_base = LLONG_MAX; - static volatile long long _Cached_Result_base = LLONG_MAX; + static volatile long long _Cached_freq = LLONG_MAX; + static volatile long long _Cached_ctr_base = LLONG_MAX; + static volatile long long _Cached_result_base = LLONG_MAX; - const long long _Freq_from_cache = _Atomic_load_ll_relaxed(&_Cached_Freq); - const long long _Ctr_base = _Atomic_load_ll_relaxed(&_Cached_Ctr_base); - const long long _Result_base = _Atomic_load_ll_relaxed(&_Cached_Result_base); + const long long _Freq_from_cache = _Atomic_load_ll_relaxed(&_Cached_freq); + const long long _Ctr_base = _Atomic_load_ll_relaxed(&_Cached_ctr_base); + const long long _Result_base = _Atomic_load_ll_relaxed(&_Cached_result_base); if (_Freq_from_cache != LLONG_MAX && _Ctr_base != LLONG_MAX && _Result_base != LLONG_MAX) { // Fast path const long long _Ctr = _Query_perf_counter(); @@ -628,10 +628,10 @@ namespace chrono { const long long _Freq = _Query_perf_frequency(); const long long _Ctr = _Query_perf_counter(); const long long _Result = _Scale_large_counter(_Ctr, _Freq); - if (_Atomic_compare_exchange_strong_ll_seq_cst(&_Cached_Freq, _Freq, LLONG_MAX)) { + if (_Atomic_compare_exchange_strong_ll_seq_cst(&_Cached_freq, _Freq, LLONG_MAX)) { // This is the first result, save current result as base for fast path - _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached_Ctr_base, _Ctr, LLONG_MAX); - _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached_Result_base, _Result, LLONG_MAX); + _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached_ctr_base, _Ctr, LLONG_MAX); + _Atomic_compare_exchange_strong_ll_seq_cst(&_Cached_result_base, _Result, LLONG_MAX); } // if _Result is not saved as first, it is still compatible with fast result return time_point(duration(_Result));