From fb59f1de6bb41efa23ac41ff9c58545118bf2e57 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 24 Mar 2025 14:49:17 -0700 Subject: [PATCH 01/14] Vectorize `find_first_not_of`/`find_last_not_of` member functions (single character) (#5102) Co-authored-by: Stephan T. Lavavej --- benchmarks/src/find_and_count.cpp | 22 ++- stl/inc/__msvc_string_view.hpp | 81 ++++++++++- stl/src/vector_algorithms.cpp | 136 ++++++++++++++---- .../VSO_0000000_vector_algorithms/test.cpp | 64 ++++++++- 4 files changed, 260 insertions(+), 43 deletions(-) diff --git a/benchmarks/src/find_and_count.cpp b/benchmarks/src/find_and_count.cpp index b0addf6a97c..448e67ac7ab 100644 --- a/benchmarks/src/find_and_count.cpp +++ b/benchmarks/src/find_and_count.cpp @@ -19,6 +19,8 @@ enum class Op { Count, StringFind, StringRFind, + StringFindNotFirstOne, + StringFindNotLastOne, }; using namespace std; @@ -28,13 +30,13 @@ void bm(benchmark::State& state) { const auto size = static_cast(state.range(0)); const auto pos = static_cast(state.range(1)); - using Container = conditional_t, Alloc>, vector>>; + using Container = + conditional_t= Op::StringFind, basic_string, Alloc>, vector>>; Container a(size, T{'0'}); if (pos < size) { - if constexpr (Operation == Op::StringRFind) { + if constexpr (Operation == Op::StringRFind || Operation == Op::StringFindNotLastOne) { a[size - pos - 1] = T{'1'}; } else { a[pos] = T{'1'}; @@ -56,6 +58,10 @@ void bm(benchmark::State& state) { benchmark::DoNotOptimize(a.find(T{'1'})); } else if constexpr (Operation == Op::StringRFind) { benchmark::DoNotOptimize(a.rfind(T{'1'})); + } else if constexpr (Operation == Op::StringFindNotFirstOne) { + benchmark::DoNotOptimize(a.find_first_not_of(T{'0'})); + } else if constexpr (Operation == Op::StringFindNotLastOne) { + benchmark::DoNotOptimize(a.find_last_not_of(T{'0'})); } } } @@ -66,27 +72,29 @@ void common_args(auto bm) { bm->Args({63, 62})->Args({31, 30})->Args({15, 14})->Args({7, 6}); } - BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); diff --git a/stl/inc/__msvc_string_view.hpp b/stl/inc/__msvc_string_view.hpp index c078de097fc..eba67768432 100644 --- a/stl/inc/__msvc_string_view.hpp +++ b/stl/inc/__msvc_string_view.hpp @@ -43,6 +43,20 @@ __declspec(noalias) size_t __stdcall __std_find_last_of_trivial_pos_1( __declspec(noalias) size_t __stdcall __std_find_last_of_trivial_pos_2( const void* _Haystack, size_t _Haystack_length, const void* _Needle, size_t _Needle_length) noexcept; +const void* __stdcall __std_find_not_ch_1(const void* _First, const void* _Last, uint8_t _Val) noexcept; +const void* __stdcall __std_find_not_ch_2(const void* _First, const void* _Last, uint16_t _Val) noexcept; +const void* __stdcall __std_find_not_ch_4(const void* _First, const void* _Last, uint32_t _Val) noexcept; +const void* __stdcall __std_find_not_ch_8(const void* _First, const void* _Last, uint64_t _Val) noexcept; + +__declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_1( + const void* _First, const void* _Last, uint8_t _Val) noexcept; +__declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_2( + const void* _First, const void* _Last, uint16_t _Val) noexcept; +__declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_4( + const void* _First, const void* _Last, uint32_t _Val) noexcept; +__declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_8( + const void* _First, const void* _Last, uint64_t _Val) noexcept; + } // extern "C" _STD_BEGIN @@ -77,6 +91,35 @@ size_t _Find_last_of_pos_vectorized(const _Ty1* const _Haystack, const size_t _H } } +template +const _Ty* _Find_not_ch_vectorized(const _Ty* const _First, const _Ty* const _Last, const _Ty _Ch) noexcept { + if constexpr (sizeof(_Ty) == 1) { + return static_cast(::__std_find_not_ch_1(_First, _Last, static_cast(_Ch))); + } else if constexpr (sizeof(_Ty) == 2) { + return static_cast(::__std_find_not_ch_2(_First, _Last, static_cast(_Ch))); + } else if constexpr (sizeof(_Ty) == 4) { + return static_cast(::__std_find_not_ch_4(_First, _Last, static_cast(_Ch))); + } else if constexpr (sizeof(_Ty) == 8) { + return static_cast(::__std_find_not_ch_8(_First, _Last, static_cast(_Ch))); + } else { + _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size + } +} + +template +size_t _Find_last_not_ch_pos_vectorized(const _Ty* const _First, const _Ty* const _Last, const _Ty _Ch) noexcept { + if constexpr (sizeof(_Ty) == 1) { + return ::__std_find_last_not_ch_pos_1(_First, _Last, static_cast(_Ch)); + } else if constexpr (sizeof(_Ty) == 2) { + return ::__std_find_last_not_ch_pos_2(_First, _Last, static_cast(_Ch)); + } else if constexpr (sizeof(_Ty) == 4) { + return ::__std_find_last_not_ch_pos_4(_First, _Last, static_cast(_Ch)); + } else if constexpr (sizeof(_Ty) == 8) { + return ::__std_find_last_not_ch_pos_8(_First, _Last, static_cast(_Ch)); + } else { + _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size + } +} _STD_END #endif // _USE_STD_VECTOR_ALGORITHMS @@ -987,14 +1030,30 @@ template constexpr size_t _Traits_find_not_ch(_In_reads_(_Hay_size) const _Traits_ptr_t<_Traits> _Haystack, const size_t _Hay_size, const size_t _Start_at, const _Traits_ch_t<_Traits> _Ch) noexcept { // search [_Haystack, _Haystack + _Hay_size) for any value other than _Ch, at/after _Start_at - if (_Start_at < _Hay_size) { // room for match, look for it - const auto _End = _Haystack + _Hay_size; - for (auto _Match_try = _Haystack + _Start_at; _Match_try < _End; ++_Match_try) { - if (!_Traits::eq(*_Match_try, _Ch)) { - return static_cast(_Match_try - _Haystack); // found a match + if (_Start_at >= _Hay_size) { // no room for match + return static_cast(-1); // no match + } + + const auto _End = _Haystack + _Hay_size; + +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (_Is_implementation_handled_char_traits<_Traits>) { + if (!_STD _Is_constant_evaluated()) { + const auto _Result = _STD _Find_not_ch_vectorized(_Haystack + _Start_at, _End, _Ch); + if (_Result != _End) { + return static_cast(_Result - _Haystack); + } else { + return static_cast(-1); // no match } } } +#endif // _USE_STD_VECTOR_ALGORITHMS + + for (auto _Match_try = _Haystack + _Start_at; _Match_try < _End; ++_Match_try) { + if (!_Traits::eq(*_Match_try, _Ch)) { + return static_cast(_Match_try - _Haystack); // found a match + } + } return static_cast(-1); // no match } @@ -1047,7 +1106,17 @@ constexpr size_t _Traits_rfind_not_ch(_In_reads_(_Hay_size) const _Traits_ptr_t< return static_cast(-1); } - for (auto _Match_try = _Haystack + (_STD min)(_Start_at, _Hay_size - 1);; --_Match_try) { + const size_t _Actual_start_at = (_STD min)(_Start_at, _Hay_size - 1); + +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (_Is_implementation_handled_char_traits<_Traits>) { + if (!_STD _Is_constant_evaluated()) { + return _STD _Find_last_not_ch_pos_vectorized(_Haystack, _Haystack + _Actual_start_at + 1, _Ch); + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + + for (auto _Match_try = _Haystack + _Actual_start_at;; --_Match_try) { if (!_Traits::eq(*_Match_try, _Ch)) { return static_cast(_Match_try - _Haystack); // found a match } diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 41c2daac2f6..15442d480d1 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2556,11 +2556,13 @@ namespace { return _Ptr; } + enum class _Find_one_predicate { _Equal, _Not_equal }; + // The below functions have exactly the same signature as the extern "C" functions, up to calling convention. // This makes sure the template specialization can be fused with the extern "C" function. // In optimized builds it avoids an extra call, as these functions are too large to inline. - template + template const void* __stdcall __std_find_trivial_impl(const void* _First, const void* _Last, _Ty _Val) noexcept { #ifndef _M_ARM64EC const size_t _Size_bytes = _Byte_length(_First, _Last); @@ -2574,7 +2576,11 @@ namespace { do { const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); - const int _Bingo = _mm256_movemask_epi8(_Traits::_Cmp_avx(_Data, _Comparand)); + int _Bingo = _mm256_movemask_epi8(_Traits::_Cmp_avx(_Data, _Comparand)); + + if constexpr (_Pred == _Find_one_predicate::_Not_equal) { + _Bingo ^= 0xFFFF'FFFF; + } if (_Bingo != 0) { const unsigned long _Offset = _tzcnt_u32(_Bingo); @@ -2588,8 +2594,11 @@ namespace { if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); - const int _Bingo = - _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + + if constexpr (_Pred == _Find_one_predicate::_Not_equal) { + _Bingo ^= (1 << _Avx_tail_size) - 1; + } if (_Bingo != 0) { const unsigned long _Offset = _tzcnt_u32(_Bingo); @@ -2610,7 +2619,11 @@ namespace { do { const __m128i _Data = _mm_loadu_si128(static_cast(_First)); - const int _Bingo = _mm_movemask_epi8(_Traits::_Cmp_sse(_Data, _Comparand)); + int _Bingo = _mm_movemask_epi8(_Traits::_Cmp_sse(_Data, _Comparand)); + + if constexpr (_Pred == _Find_one_predicate::_Not_equal) { + _Bingo ^= 0xFFFF; + } if (_Bingo != 0) { unsigned long _Offset; @@ -2625,13 +2638,19 @@ namespace { } #endif // !_M_ARM64EC auto _Ptr = static_cast(_First); - while (_Ptr != _Last && *_Ptr != _Val) { - ++_Ptr; + if constexpr (_Pred == _Find_one_predicate::_Not_equal) { + while (_Ptr != _Last && *_Ptr == _Val) { + ++_Ptr; + } + } else { + while (_Ptr != _Last && *_Ptr != _Val) { + ++_Ptr; + } } return _Ptr; } - template + template const void* __stdcall __std_find_last_trivial_impl(const void* _First, const void* _Last, _Ty _Val) noexcept { const void* const _Real_last = _Last; #ifndef _M_ARM64EC @@ -2647,7 +2666,11 @@ namespace { do { _Rewind_bytes(_Last, 32); const __m256i _Data = _mm256_loadu_si256(static_cast(_Last)); - const int _Bingo = _mm256_movemask_epi8(_Traits::_Cmp_avx(_Data, _Comparand)); + int _Bingo = _mm256_movemask_epi8(_Traits::_Cmp_avx(_Data, _Comparand)); + + if constexpr (_Pred == _Find_one_predicate::_Not_equal) { + _Bingo ^= 0xFFFF'FFFF; + } if (_Bingo != 0) { const unsigned long _Offset = _lzcnt_u32(_Bingo); @@ -2660,8 +2683,11 @@ namespace { _Rewind_bytes(_Last, _Avx_tail_size); const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); const __m256i _Data = _mm256_maskload_epi32(static_cast(_Last), _Tail_mask); - const int _Bingo = - _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + int _Bingo = _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + + if constexpr (_Pred == _Find_one_predicate::_Not_equal) { + _Bingo ^= (1 << _Avx_tail_size) - 1; + } if (_Bingo != 0) { const unsigned long _Offset = _lzcnt_u32(_Bingo); @@ -2681,7 +2707,11 @@ namespace { do { _Rewind_bytes(_Last, 16); const __m128i _Data = _mm_loadu_si128(static_cast(_Last)); - const int _Bingo = _mm_movemask_epi8(_Traits::_Cmp_sse(_Data, _Comparand)); + int _Bingo = _mm_movemask_epi8(_Traits::_Cmp_sse(_Data, _Comparand)); + + if constexpr (_Pred == _Find_one_predicate::_Not_equal) { + _Bingo ^= 0xFFFF; + } if (_Bingo != 0) { unsigned long _Offset; @@ -2699,12 +2729,28 @@ namespace { return _Real_last; } --_Ptr; - if (*_Ptr == _Val) { - return _Ptr; + if constexpr (_Pred == _Find_one_predicate::_Not_equal) { + if (*_Ptr != _Val) { + return _Ptr; + } + } else { + if (*_Ptr == _Val) { + return _Ptr; + } } } } + template + size_t __stdcall __std_find_last_pos(const void* const _First, const void* const _Last, const _Ty _Val) noexcept { + const void* const _Result = __std_find_last_trivial_impl<_Traits, _Pred>(_First, _Last, _Val); + if (_Result == _Last) { + return static_cast(-1); + } else { + return _Byte_length(_First, _Result) / sizeof(_Ty); + } + } + struct _Count_traits_8 : _Find_traits_8 { #ifndef _M_ARM64EC static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { @@ -4089,7 +4135,8 @@ namespace { } if (_Count2 == 1) { - return __std_find_trivial_impl<_Traits>(_First1, _Last1, *static_cast(_First2)); + return __std_find_trivial_impl<_Traits, _Find_one_predicate::_Equal>( + _First1, _Last1, *static_cast(_First2)); } const size_t _Size_bytes_1 = _Byte_length(_First1, _Last1); @@ -4238,7 +4285,8 @@ namespace { } if (_Count2 == 1) { - return __std_find_last_trivial_impl<_Traits>(_First1, _Last1, *static_cast(_First2)); + return __std_find_last_trivial_impl<_Traits, _Find_one_predicate::_Equal>( + _First1, _Last1, *static_cast(_First2)); } const size_t _Size_bytes_1 = _Byte_length(_First1, _Last1); @@ -4507,42 +4555,82 @@ const void* __stdcall __std_find_trivial_unsized_8(const void* const _First, con const void* __stdcall __std_find_trivial_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { - return __std_find_trivial_impl<_Find_traits_1>(_First, _Last, _Val); + return __std_find_trivial_impl<_Find_traits_1, _Find_one_predicate::_Equal>(_First, _Last, _Val); } const void* __stdcall __std_find_trivial_2( const void* const _First, const void* const _Last, const uint16_t _Val) noexcept { - return __std_find_trivial_impl<_Find_traits_2>(_First, _Last, _Val); + return __std_find_trivial_impl<_Find_traits_2, _Find_one_predicate::_Equal>(_First, _Last, _Val); } const void* __stdcall __std_find_trivial_4( const void* const _First, const void* const _Last, const uint32_t _Val) noexcept { - return __std_find_trivial_impl<_Find_traits_4>(_First, _Last, _Val); + return __std_find_trivial_impl<_Find_traits_4, _Find_one_predicate::_Equal>(_First, _Last, _Val); } const void* __stdcall __std_find_trivial_8( const void* const _First, const void* const _Last, const uint64_t _Val) noexcept { - return __std_find_trivial_impl<_Find_traits_8>(_First, _Last, _Val); + return __std_find_trivial_impl<_Find_traits_8, _Find_one_predicate::_Equal>(_First, _Last, _Val); } const void* __stdcall __std_find_last_trivial_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { - return __std_find_last_trivial_impl<_Find_traits_1>(_First, _Last, _Val); + return __std_find_last_trivial_impl<_Find_traits_1, _Find_one_predicate::_Equal>(_First, _Last, _Val); } const void* __stdcall __std_find_last_trivial_2( const void* const _First, const void* const _Last, const uint16_t _Val) noexcept { - return __std_find_last_trivial_impl<_Find_traits_2>(_First, _Last, _Val); + return __std_find_last_trivial_impl<_Find_traits_2, _Find_one_predicate::_Equal>(_First, _Last, _Val); } const void* __stdcall __std_find_last_trivial_4( const void* const _First, const void* const _Last, const uint32_t _Val) noexcept { - return __std_find_last_trivial_impl<_Find_traits_4>(_First, _Last, _Val); + return __std_find_last_trivial_impl<_Find_traits_4, _Find_one_predicate::_Equal>(_First, _Last, _Val); } const void* __stdcall __std_find_last_trivial_8( const void* const _First, const void* const _Last, const uint64_t _Val) noexcept { - return __std_find_last_trivial_impl<_Find_traits_8>(_First, _Last, _Val); + return __std_find_last_trivial_impl<_Find_traits_8, _Find_one_predicate::_Equal>(_First, _Last, _Val); +} + +const void* __stdcall __std_find_not_ch_1( + const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { + return __std_find_trivial_impl<_Find_traits_1, _Find_one_predicate::_Not_equal>(_First, _Last, _Val); +} + +const void* __stdcall __std_find_not_ch_2( + const void* const _First, const void* const _Last, const uint16_t _Val) noexcept { + return __std_find_trivial_impl<_Find_traits_2, _Find_one_predicate::_Not_equal>(_First, _Last, _Val); +} + +const void* __stdcall __std_find_not_ch_4( + const void* const _First, const void* const _Last, const uint32_t _Val) noexcept { + return __std_find_trivial_impl<_Find_traits_4, _Find_one_predicate::_Not_equal>(_First, _Last, _Val); +} + +const void* __stdcall __std_find_not_ch_8( + const void* const _First, const void* const _Last, const uint64_t _Val) noexcept { + return __std_find_trivial_impl<_Find_traits_8, _Find_one_predicate::_Not_equal>(_First, _Last, _Val); +} + +__declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_1( + const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { + return __std_find_last_pos<_Find_traits_1, _Find_one_predicate::_Not_equal>(_First, _Last, _Val); +} + +__declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_2( + const void* const _First, const void* const _Last, const uint16_t _Val) noexcept { + return __std_find_last_pos<_Find_traits_2, _Find_one_predicate::_Not_equal>(_First, _Last, _Val); +} + +__declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_4( + const void* const _First, const void* const _Last, const uint32_t _Val) noexcept { + return __std_find_last_pos<_Find_traits_4, _Find_one_predicate::_Not_equal>(_First, _Last, _Val); +} + +__declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_8( + const void* const _First, const void* const _Last, const uint64_t _Val) noexcept { + return __std_find_last_pos<_Find_traits_8, _Find_one_predicate::_Not_equal>(_First, _Last, _Val); } __declspec(noalias) size_t __stdcall __std_count_trivial_1( diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 293ee4520e9..9ccaadf6ea3 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -171,25 +171,25 @@ void test_count(mt19937_64& gen) { } } -template -auto last_known_good_find(FwdIt first, FwdIt last, T v) { +template > +auto last_known_good_find(FwdIt first, FwdIt last, T v, Pred pred = {}) { for (; first != last; ++first) { - if (*first == v) { + if (pred(*first, v)) { break; } } return first; } -template -auto last_known_good_find_last(FwdIt first, FwdIt last, T v) { +template > +auto last_known_good_find_last(FwdIt first, FwdIt last, T v, Pred pred = {}) { FwdIt last_save = last; for (;;) { if (last == first) { return last_save; } --last; - if (*last == v) { + if (pred(*last, v)) { return last; } } @@ -1162,12 +1162,48 @@ void test_case_string_rfind_str(const basic_string& input_haystack, const bas assert(expected == actual); } +template +void test_case_string_find_not_ch(const basic_string& input_haystack, const T value) { + ptrdiff_t expected; + + const auto expected_iter = + last_known_good_find(input_haystack.begin(), input_haystack.end(), value, not_equal_to<>{}); + + if (expected_iter != input_haystack.end()) { + expected = expected_iter - input_haystack.begin(); + } else { + expected = -1; + } + + const auto actual = static_cast(input_haystack.find_first_not_of(value)); + assert(expected == actual); +} + +template +void test_case_string_rfind_not_ch(const basic_string& input_haystack, const T value) { + ptrdiff_t expected; + + const auto expected_iter = + last_known_good_find_last(input_haystack.begin(), input_haystack.end(), value, not_equal_to<>{}); + + if (expected_iter != input_haystack.end()) { + expected = expected_iter - input_haystack.begin(); + } else { + expected = -1; + } + + const auto actual = static_cast(input_haystack.find_last_not_of(value)); + assert(expected == actual); +} + template void test_basic_string_dis(mt19937_64& gen, D& dis) { basic_string input_haystack; + basic_string input_haystack_not; basic_string input_needle; basic_string temp; input_haystack.reserve(haystackDataCount); + input_haystack_not.reserve(haystackDataCount); input_needle.reserve(needleDataCount); temp.reserve(needleDataCount); @@ -1203,6 +1239,22 @@ void test_basic_string_dis(mt19937_64& gen, D& dis) { } } + const auto input_not_ch = static_cast(dis(gen)); + input_haystack_not.assign(input_haystack.size(), input_not_ch); + + test_case_string_find_not_ch(input_haystack_not, input_not_ch); + test_case_string_rfind_not_ch(input_haystack_not, input_not_ch); + if (!input_haystack_not.empty()) { + uniform_int_distribution not_pos_dis(0, input_haystack_not.size() - 1); + + for (size_t attempts = 0; attempts < needleDataCount; ++attempts) { + const size_t pos = not_pos_dis(gen); + input_haystack_not[pos] = input_haystack[pos]; + test_case_string_find_not_ch(input_haystack_not, input_not_ch); + test_case_string_rfind_not_ch(input_haystack_not, input_not_ch); + } + } + if (input_haystack.size() == haystackDataCount) { break; } From 185398a6ed2b5e616550e7ffc79eabffc025ffd7 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 24 Mar 2025 15:41:04 -0700 Subject: [PATCH 02/14] Vectorize `find_first_not_of`/`find_last_not_of` member functions (multiple characters overloads) (#5206) Co-authored-by: Stephan T. Lavavej --- benchmarks/src/find_first_of.cpp | 56 ++- stl/inc/__msvc_string_view.hpp | 62 +++ stl/src/vector_algorithms.cpp | 387 +++++++++++++----- .../VSO_0000000_vector_algorithms/test.cpp | 63 ++- 4 files changed, 447 insertions(+), 121 deletions(-) diff --git a/benchmarks/src/find_first_of.cpp b/benchmarks/src/find_first_of.cpp index 41b2089e4ca..bf6a8585f02 100644 --- a/benchmarks/src/find_first_of.cpp +++ b/benchmarks/src/find_first_of.cpp @@ -5,10 +5,7 @@ #include #include #include -#include #include -#include -#include #include #include #include @@ -17,9 +14,15 @@ using namespace std; -enum class AlgType { std_func, str_member_first, str_member_last }; +enum class AlgType { + std_func, + str_member_first, + str_member_last, + str_member_first_not, + str_member_last_not, +}; -template +template void bm(benchmark::State& state) { const size_t Pos = static_cast(state.range(0)); const size_t NSize = static_cast(state.range(1)); @@ -29,24 +32,37 @@ void bm(benchmark::State& state) { using container = conditional_t>, basic_string, not_highly_aligned_allocator>>; - constexpr T HaystackFiller{' '}; - static_assert(HaystackFiller < Start, "The following iota() should not produce the haystack filler."); + constexpr size_t IncrementCap = 16; - container h(HSize, HaystackFiller); + constexpr T HaystackFillerBase = T{' '}; + static_assert( + NeedleFillerBase + IncrementCap <= HaystackFillerBase || HaystackFillerBase + IncrementCap <= NeedleFillerBase, + "Would match where it shouldn't"); + + container h(HSize, T{0}); container n(NSize, T{0}); - if (NSize - 1 > static_cast(numeric_limits::max()) - static_cast(Start)) { - puts("ERROR: The following iota() would overflow."); - abort(); + for (size_t i = 0; i != NSize; ++i) { + n[i] = NeedleFillerBase + i % IncrementCap; } - iota(n.begin(), n.end(), Start); - if (Pos >= HSize || Which >= NSize) { abort(); } - h[Pos] = n[Which]; + if constexpr (Alg == AlgType::str_member_first_not || Alg == AlgType::str_member_last_not) { + for (size_t i = 0; i != HSize; ++i) { + h[i] = n[(i + Which) % NSize]; + } + + h[Pos] = HaystackFillerBase; + } else { + for (size_t i = 0; i != HSize; ++i) { + h[i] = HaystackFillerBase + i % IncrementCap; + } + + h[Pos] = n[Which]; + } for (auto _ : state) { benchmark::DoNotOptimize(h); @@ -55,6 +71,10 @@ void bm(benchmark::State& state) { benchmark::DoNotOptimize(h.find_first_of(n)); } else if constexpr (Alg == AlgType::str_member_last) { benchmark::DoNotOptimize(h.find_last_of(n)); + } else if constexpr (Alg == AlgType::str_member_first_not) { + benchmark::DoNotOptimize(h.find_first_not_of(n)); + } else if constexpr (Alg == AlgType::str_member_last_not) { + benchmark::DoNotOptimize(h.find_last_not_of(n)); } else { benchmark::DoNotOptimize(find_first_of(h.begin(), h.end(), n.begin(), n.end())); } @@ -82,4 +102,12 @@ BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + BENCHMARK_MAIN(); diff --git a/stl/inc/__msvc_string_view.hpp b/stl/inc/__msvc_string_view.hpp index eba67768432..d6e37a6d46f 100644 --- a/stl/inc/__msvc_string_view.hpp +++ b/stl/inc/__msvc_string_view.hpp @@ -57,6 +57,16 @@ __declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_4( __declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_8( const void* _First, const void* _Last, uint64_t _Val) noexcept; +__declspec(noalias) size_t __stdcall __std_find_first_not_of_trivial_pos_1( + const void* _Haystack, size_t _Haystack_length, const void* _Needle, size_t _Needle_length) noexcept; +__declspec(noalias) size_t __stdcall __std_find_first_not_of_trivial_pos_2( + const void* _Haystack, size_t _Haystack_length, const void* _Needle, size_t _Needle_length) noexcept; + +__declspec(noalias) size_t __stdcall __std_find_last_not_of_trivial_pos_1( + const void* _Haystack, size_t _Haystack_length, const void* _Needle, size_t _Needle_length) noexcept; +__declspec(noalias) size_t __stdcall __std_find_last_not_of_trivial_pos_2( + const void* _Haystack, size_t _Haystack_length, const void* _Needle, size_t _Needle_length) noexcept; + } // extern "C" _STD_BEGIN @@ -120,6 +130,32 @@ size_t _Find_last_not_ch_pos_vectorized(const _Ty* const _First, const _Ty* cons _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size } } +template +size_t _Find_first_not_of_pos_vectorized(const _Ty1* const _Haystack, const size_t _Haystack_length, + const _Ty2* const _Needle, const size_t _Needle_length) noexcept { + _STL_INTERNAL_STATIC_ASSERT(sizeof(_Ty1) == sizeof(_Ty2)); + if constexpr (sizeof(_Ty1) == 1) { + return ::__std_find_first_not_of_trivial_pos_1(_Haystack, _Haystack_length, _Needle, _Needle_length); + } else if constexpr (sizeof(_Ty1) == 2) { + return ::__std_find_first_not_of_trivial_pos_2(_Haystack, _Haystack_length, _Needle, _Needle_length); + } else { + _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size + } +} + +template +size_t _Find_last_not_of_pos_vectorized(const _Ty1* const _Haystack, const size_t _Haystack_length, + const _Ty2* const _Needle, const size_t _Needle_length) noexcept { + _STL_INTERNAL_STATIC_ASSERT(sizeof(_Ty1) == sizeof(_Ty2)); + if constexpr (sizeof(_Ty1) == 1) { + return ::__std_find_last_not_of_trivial_pos_1(_Haystack, _Haystack_length, _Needle, _Needle_length); + } else if constexpr (sizeof(_Ty1) == 2) { + return ::__std_find_last_not_of_trivial_pos_2(_Haystack, _Haystack_length, _Needle, _Needle_length); + } else { + _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size + } +} + _STD_END #endif // _USE_STD_VECTOR_ALGORITHMS @@ -1004,6 +1040,21 @@ constexpr size_t _Traits_find_first_not_of(_In_reads_(_Hay_size) const _Traits_p if constexpr (_Is_implementation_handled_char_traits<_Traits>) { using _Elem = typename _Traits::char_type; +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (sizeof(_Elem) <= 2) { + if (!_STD _Is_constant_evaluated()) { + const size_t _Remaining_size = _Hay_size - _Start_at; + if (_Remaining_size + _Needle_size >= _Threshold_find_first_of) { + size_t _Pos = _Find_first_not_of_pos_vectorized(_Hay_start, _Remaining_size, _Needle, _Needle_size); + if (_Pos != static_cast(-1)) { + _Pos += _Start_at; + } + return _Pos; + } + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + _String_bitmap<_Elem> _Matches; if (_Matches._Mark(_Needle, _Needle + _Needle_size)) { for (auto _Match_try = _Hay_start; _Match_try < _Hay_end; ++_Match_try) { @@ -1071,6 +1122,17 @@ constexpr size_t _Traits_find_last_not_of(_In_reads_(_Hay_size) const _Traits_pt if constexpr (_Is_implementation_handled_char_traits<_Traits>) { using _Elem = typename _Traits::char_type; +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (sizeof(_Elem) <= 2) { + if (!_STD _Is_constant_evaluated()) { + const size_t _Remaining_size = _Hay_start + 1; + if (_Remaining_size + _Needle_size >= _Threshold_find_first_of) { // same threshold for first/last + return _Find_last_not_of_pos_vectorized(_Haystack, _Remaining_size, _Needle, _Needle_size); + } + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + _String_bitmap<_Elem> _Matches; if (_Matches._Mark(_Needle, _Needle + _Needle_size)) { for (auto _Match_try = _Haystack + _Hay_start;; --_Match_try) { diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 15442d480d1..559de319996 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2976,6 +2976,8 @@ namespace { return _Result; } + enum class _Find_meow_of_predicate { _Any_of, _None_of }; + #ifndef _M_ARM64EC namespace __std_find_meow_of_bitmap_details { __m256i _Bitmap_step(const __m256i _Bitmap, const __m256i _Data) noexcept { @@ -3246,7 +3248,7 @@ namespace { } } - template + template size_t _Impl_first_avx(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { using namespace __std_find_meow_of_bitmap_details; @@ -3260,9 +3262,14 @@ namespace { const size_t _Haystack_length_vec = _Haystack_length & ~size_t{7}; for (size_t _Ix = 0; _Ix != _Haystack_length_vec; _Ix += 8) { - const __m256i _Data = _Load_avx_256_8(_Haystack_ptr + _Ix); - const __m256i _Mask = _Mask_out_overflow<_Ty>(_Bitmap_step(_Bitmap, _Data), _Data); - const unsigned int _Bingo = _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)); + const __m256i _Data = _Load_avx_256_8(_Haystack_ptr + _Ix); + const __m256i _Mask = _Mask_out_overflow<_Ty>(_Bitmap_step(_Bitmap, _Data), _Data); + unsigned int _Bingo = _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)); + + if constexpr (_Pred == _Find_meow_of_predicate::_None_of) { + _Bingo ^= 0xFF; + } + if (_Bingo != 0) { return _Ix + _tzcnt_u32(_Bingo); } @@ -3273,7 +3280,12 @@ namespace { const unsigned int _Tail_bingo_mask = (1 << _Haystack_length_tail) - 1; const __m256i _Data = _Load_avx_256_8_last(_Haystack_ptr + _Haystack_length_vec, _Haystack_length_tail); const __m256i _Mask = _Mask_out_overflow<_Ty>(_Bitmap_step(_Bitmap, _Data), _Data); - const unsigned int _Bingo = _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)) & _Tail_bingo_mask; + unsigned int _Bingo = _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)) & _Tail_bingo_mask; + + if constexpr (_Pred == _Find_meow_of_predicate::_None_of) { + _Bingo ^= _Tail_bingo_mask; + } + if (_Bingo != 0) { return _Haystack_length_vec + _tzcnt_u32(_Bingo); } @@ -3282,7 +3294,7 @@ namespace { return static_cast(-1); } - template + template size_t _Impl_last_avx(const void* const _Haystack, size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { using namespace __std_find_meow_of_bitmap_details; @@ -3296,9 +3308,14 @@ namespace { while (_Haystack_length >= 8) { _Haystack_length -= 8; - const __m256i _Data = _Load_avx_256_8(_Haystack_ptr + _Haystack_length); - const __m256i _Mask = _Mask_out_overflow<_Ty>(_Bitmap_step(_Bitmap, _Data), _Data); - const unsigned int _Bingo = _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)); + const __m256i _Data = _Load_avx_256_8(_Haystack_ptr + _Haystack_length); + const __m256i _Mask = _Mask_out_overflow<_Ty>(_Bitmap_step(_Bitmap, _Data), _Data); + unsigned int _Bingo = _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)); + + if constexpr (_Pred == _Find_meow_of_predicate::_None_of) { + _Bingo ^= 0xFF; + } + if (_Bingo != 0) { return _Haystack_length + 31 - _lzcnt_u32(_Bingo); } @@ -3309,7 +3326,12 @@ namespace { const unsigned int _Tail_bingo_mask = (1 << _Haystack_length_tail) - 1; const __m256i _Data = _Load_avx_256_8_last(_Haystack_ptr, _Haystack_length_tail); const __m256i _Mask = _Mask_out_overflow<_Ty>(_Bitmap_step(_Bitmap, _Data), _Data); - const unsigned int _Bingo = _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)) & _Tail_bingo_mask; + unsigned int _Bingo = _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)) & _Tail_bingo_mask; + + if constexpr (_Pred == _Find_meow_of_predicate::_None_of) { + _Bingo ^= _Tail_bingo_mask; + } + if (_Bingo != 0) { return 31 - _lzcnt_u32(_Bingo); } @@ -3355,7 +3377,7 @@ namespace { } #endif // !_M_ARM64EC - template + template size_t _Impl_first_scalar( const void* const _Haystack, const size_t _Haystack_length, const _Scalar_table_t& _Table) noexcept { const auto _Haystack_ptr = static_cast(_Haystack); @@ -3365,19 +3387,29 @@ namespace { if constexpr (sizeof(_Val) > 1) { if (_Val >= 256) { - continue; + if constexpr (_Pred == _Find_meow_of_predicate::_Any_of) { + continue; + } else { + return _Ix; + } } } - if (_Table[_Val]) { - return _Ix; + if constexpr (_Pred == _Find_meow_of_predicate::_Any_of) { + if (_Table[_Val]) { + return _Ix; + } + } else { + if (!_Table[_Val]) { + return _Ix; + } } } return static_cast(-1); } - template + template size_t _Impl_last_scalar( const void* const _Haystack, size_t _Haystack_length, const _Scalar_table_t& _Table) noexcept { const auto _Haystack_ptr = static_cast(_Haystack); @@ -3389,12 +3421,22 @@ namespace { if constexpr (sizeof(_Val) > 1) { if (_Val >= 256) { - continue; + if constexpr (_Pred == _Find_meow_of_predicate::_Any_of) { + continue; + } else { + return _Haystack_length; + } } } - if (_Table[_Val]) { - return _Haystack_length; + if constexpr (_Pred == _Find_meow_of_predicate::_Any_of) { + if (_Table[_Val]) { + return _Haystack_length; + } + } else { + if (!_Table[_Val]) { + return _Haystack_length; + } } } @@ -3403,7 +3445,7 @@ namespace { } // namespace __std_find_meow_of_bitmap namespace __std_find_first_of { - template + template const void* _Fallback(const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { auto _Ptr_haystack = static_cast(_First1); @@ -3412,8 +3454,22 @@ namespace { const auto _Ptr_needle_end = static_cast(_Last2); for (; _Ptr_haystack != _Ptr_haystack_end; ++_Ptr_haystack) { - for (auto _Ptr = _Ptr_needle; _Ptr != _Ptr_needle_end; ++_Ptr) { - if (*_Ptr_haystack == *_Ptr) { + if constexpr (_Pred == _Find_meow_of_predicate::_Any_of) { + for (auto _Ptr = _Ptr_needle; _Ptr != _Ptr_needle_end; ++_Ptr) { + if (*_Ptr_haystack == *_Ptr) { + return _Ptr_haystack; + } + } + } else { + bool _Match = false; + for (auto _Ptr = _Ptr_needle; _Ptr != _Ptr_needle_end; ++_Ptr) { + if (*_Ptr_haystack == *_Ptr) { + _Match = true; + break; + } + } + + if (!_Match) { return _Ptr_haystack; } } @@ -3423,11 +3479,13 @@ namespace { } #ifndef _M_ARM64EC - template + template const void* _Impl_pcmpestri(const void* _First1, const size_t _Haystack_length, const void* const _First2, const size_t _Needle_length) noexcept { - constexpr int _Op = - (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; + constexpr int _Op_base = + (_Pred == _Find_meow_of_predicate::_Any_of ? _SIDD_POSITIVE_POLARITY : _SIDD_MASKED_NEGATIVE_POLARITY) + | (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY; + constexpr int _Op = _Op_base | _SIDD_LEAST_SIGNIFICANT; constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; const void* _Stop_at = _First1; @@ -3483,37 +3541,78 @@ namespace { const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); constexpr int _Not_found = 16; // arbitrary value greater than any found value +#pragma warning(push) +#pragma warning(disable : 4324) // structure was padded due to alignment specifier + const auto _Test_whole_needle = [=](const __m128i _Data1, const int _Size1, + const int _Found_pos_init) noexcept { + if constexpr (_Pred == _Find_meow_of_predicate::_Any_of) { + int _Found_pos = _Found_pos_init; + + const auto _Step = [&_Found_pos](const __m128i _Data2, const int _Size2, const __m128i _Data1, + const int _Size1) noexcept { + if (_mm_cmpestrc(_Data2, _Size2, _Data1, _Size1, _Op)) { + const int _Pos = _mm_cmpestri(_Data2, _Size2, _Data1, _Size1, _Op); + if (_Pos < _Found_pos) { + _Found_pos = _Pos; + } + } + }; - int _Found_pos = _Not_found; + const void* _Cur_needle = _First2; + do { + const __m128i _Data2 = _mm_loadu_si128(static_cast(_Cur_needle)); + _Step(_Data2, _Part_size_el, _Data1, _Size1); + _Advance_bytes(_Cur_needle, 16); + } while (_Cur_needle != _Last_needle); - const auto _Step = [&_Found_pos](const __m128i _Data2, const int _Size2, const __m128i _Data1, - const int _Size1) noexcept { - if (_mm_cmpestrc(_Data2, _Size2, _Data1, _Size1, _Op)) { - const int _Pos = _mm_cmpestri(_Data2, _Size2, _Data1, _Size1, _Op); - if (_Pos < _Found_pos) { - _Found_pos = _Pos; + if (_Last_needle_length_el != 0) { + _Step(_Last_needle_val, _Last_needle_length_el, _Data1, _Size1); } - } - }; -#pragma warning(push) -#pragma warning(disable : 4324) // structure was padded due to alignment specifier - const auto _Test_whole_needle = [=](const __m128i _Data1, const int _Size1) noexcept { - const void* _Cur_needle = _First2; - do { - const __m128i _Data2 = _mm_loadu_si128(static_cast(_Cur_needle)); - _Step(_Data2, _Part_size_el, _Data1, _Size1); + return _Found_pos; + } else { + constexpr int _Op_mask = _Op_base | _SIDD_BIT_MASK; + + const void* _Cur_needle = _First2; + + const __m128i _Data2_first = _mm_loadu_si128(static_cast(_Cur_needle)); + + __m128i _Found = _mm_cmpestrm(_Data2_first, _Part_size_el, _Data1, _Size1, _Op_mask); _Advance_bytes(_Cur_needle, 16); - } while (_Cur_needle != _Last_needle); - if (_Last_needle_length_el != 0) { - _Step(_Last_needle_val, _Last_needle_length_el, _Data1, _Size1); + while (_Cur_needle != _Last_needle) { + const __m128i _Data2 = _mm_loadu_si128(static_cast(_Cur_needle)); + const __m128i _Found_part = _mm_cmpestrm(_Data2, _Part_size_el, _Data1, _Size1, _Op_mask); + _Found = _mm_and_si128(_Found, _Found_part); + _Advance_bytes(_Cur_needle, 16); + } + + if (_Last_needle_length_el != 0) { + const __m128i _Found_part = + _mm_cmpestrm(_Last_needle_val, _Last_needle_length_el, _Data1, _Size1, _Op_mask); + _Found = _mm_and_si128(_Found, _Found_part); + } + + const int _Bingo = _mm_cvtsi128_si32(_Found); + int _Found_pos = _Found_pos_init; + + if (_Bingo != 0) { + unsigned long _Tmp; + // CodeQL [SM02313] _Tmp is always initialized: we just tested `if (_Bingo != 0)`. + _BitScanForward(&_Tmp, _Bingo); + if (_Found_pos > static_cast(_Tmp)) { + _Found_pos = static_cast(_Tmp); + } + } + + return _Found_pos; } }; #pragma warning(pop) while (_First1 != _Stop_at) { - _Test_whole_needle(_mm_loadu_si128(static_cast(_First1)), _Part_size_el); + const int _Found_pos = _Test_whole_needle( + _mm_loadu_si128(static_cast(_First1)), _Part_size_el, _Not_found); if (_Found_pos != _Not_found) { _Advance_bytes(_First1, _Found_pos * sizeof(_Ty)); @@ -3530,9 +3629,7 @@ namespace { memcpy(_Tmp1, _First1, _Last_part_size); const __m128i _Data1 = _mm_load_si128(reinterpret_cast(_Tmp1)); - _Found_pos = _Last_part_size_el; - - _Test_whole_needle(_Data1, _Last_part_size_el); + const int _Found_pos = _Test_whole_needle(_Data1, _Last_part_size_el, _Last_part_size_el); _Advance_bytes(_First1, _Found_pos * sizeof(_Ty)); } @@ -3757,7 +3854,7 @@ namespace { #ifndef _M_ARM64EC if constexpr (sizeof(_Ty) <= 2) { if (_Use_sse42()) { - return _Impl_pcmpestri<_Ty>( + return _Impl_pcmpestri<_Ty, _Find_meow_of_predicate::_Any_of>( _First1, _Byte_length(_First1, _Last1), _First2, _Byte_length(_First2, _Last2)); } } else { @@ -3768,7 +3865,7 @@ namespace { } #endif // !_M_ARM64EC - return _Fallback<_Ty>(_First1, _Last1, _First2, _Last2); + return _Fallback<_Ty, _Find_meow_of_predicate::_Any_of>(_First1, _Last1, _First2, _Last2); } template @@ -3781,7 +3878,7 @@ namespace { } #ifndef _M_ARM64EC - template + template size_t _Dispatch_pos_sse_1_2( const void* const _First1, const size_t _Count1, const void* const _First2, const size_t _Count2) noexcept { using namespace __std_find_meow_of_bitmap; @@ -3790,13 +3887,13 @@ namespace { if (_Strat == _Strategy::_Vector_bitmap) { if (_Can_fit_256_bits_sse(static_cast(_First2), _Count2)) { - return _Impl_first_avx<_Ty>(_First1, _Count1, _First2, _Count2); + return _Impl_first_avx<_Ty, _Pred>(_First1, _Count1, _First2, _Count2); } } else if (_Strat == _Strategy::_Scalar_bitmap) { if (_Can_fit_256_bits_sse(static_cast(_First2), _Count2)) { alignas(32) _Scalar_table_t _Table = {}; _Build_scalar_table_no_check<_Ty>(_First2, _Count2, _Table); - return _Impl_first_scalar<_Ty>(_First1, _Count1, _Table); + return _Impl_first_scalar<_Ty, _Pred>(_First1, _Count1, _Table); } } @@ -3805,7 +3902,7 @@ namespace { const size_t _Size_bytes_2 = _Count2 * sizeof(_Ty); return _Pos_from_ptr<_Ty>( - _Impl_pcmpestri<_Ty>(_First1, _Size_bytes_1, _First2, _Size_bytes_2), _First1, _Last1); + _Impl_pcmpestri<_Ty, _Pred>(_First1, _Size_bytes_1, _First2, _Size_bytes_2), _First1, _Last1); } template @@ -3817,13 +3914,13 @@ namespace { if (_Strat == _Strategy::_Vector_bitmap) { if (_Can_fit_256_bits_sse(static_cast(_First2), _Count2)) { - return _Impl_first_avx<_Ty>(_First1, _Count1, _First2, _Count2); + return _Impl_first_avx<_Ty, _Find_meow_of_predicate::_Any_of>(_First1, _Count1, _First2, _Count2); } } else if (_Strat == _Strategy::_Scalar_bitmap) { if (_Can_fit_256_bits_sse(static_cast(_First2), _Count2)) { alignas(32) _Scalar_table_t _Table = {}; _Build_scalar_table_no_check<_Ty>(_First2, _Count2, _Table); - return _Impl_first_scalar<_Ty>(_First1, _Count1, _Table); + return _Impl_first_scalar<_Ty, _Find_meow_of_predicate::_Any_of>(_First1, _Count1, _Table); } } @@ -3835,42 +3932,44 @@ namespace { } #endif // !_M_ARM64EC - template + template size_t _Dispatch_pos_fallback( const void* const _First1, const size_t _Count1, const void* const _First2, const size_t _Count2) noexcept { using namespace __std_find_meow_of_bitmap; _Scalar_table_t _Table = {}; if (_Build_scalar_table<_Ty>(_First2, _Count2, _Table)) { - return _Impl_first_scalar<_Ty>(_First1, _Count1, _Table); + return _Impl_first_scalar<_Ty, _Pred>(_First1, _Count1, _Table); } const void* const _Last1 = static_cast(_First1) + _Count1; const void* const _Last2 = static_cast(_First2) + _Count2; - return _Pos_from_ptr<_Ty>(_Fallback<_Ty>(_First1, _Last1, _First2, _Last2), _First1, _Last1); + return _Pos_from_ptr<_Ty>(_Fallback<_Ty, _Pred>(_First1, _Last1, _First2, _Last2), _First1, _Last1); } - template + template size_t _Dispatch_pos( const void* const _First1, const size_t _Count1, const void* const _First2, const size_t _Count2) noexcept { #ifndef _M_ARM64EC if constexpr (sizeof(_Ty) <= 2) { if (_Use_sse42()) { - return _Dispatch_pos_sse_1_2<_Ty>(_First1, _Count1, _First2, _Count2); + return _Dispatch_pos_sse_1_2<_Ty, _Pred>(_First1, _Count1, _First2, _Count2); } } else { if (_Use_avx2()) { + static_assert(_Pred == _Find_meow_of_predicate::_Any_of); + return _Dispatch_pos_avx_4_8<_Ty>(_First1, _Count1, _First2, _Count2); } } #endif // !_M_ARM64EC - return _Dispatch_pos_fallback<_Ty>(_First1, _Count1, _First2, _Count2); + return _Dispatch_pos_fallback<_Ty, _Pred>(_First1, _Count1, _First2, _Count2); } } // namespace __std_find_first_of namespace __std_find_last_of { - template + template size_t __stdcall _Fallback(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { @@ -3881,8 +3980,22 @@ namespace { while (_Pos != 0) { --_Pos; - for (auto _Ptr = static_cast(_Needle); _Ptr != _Needle_end; ++_Ptr) { - if (_Ptr_haystack[_Pos] == *_Ptr) { + if constexpr (_Pred == _Find_meow_of_predicate::_Any_of) { + for (auto _Ptr = static_cast(_Needle); _Ptr != _Needle_end; ++_Ptr) { + if (_Ptr_haystack[_Pos] == *_Ptr) { + return _Pos; + } + } + } else { + bool _Match = false; + for (auto _Ptr = static_cast(_Needle); _Ptr != _Needle_end; ++_Ptr) { + if (_Ptr_haystack[_Pos] == *_Ptr) { + _Match = true; + break; + } + } + + if (!_Match) { return _Pos; } } @@ -3892,13 +4005,15 @@ namespace { } #ifndef _M_ARM64EC - template + template size_t _Impl(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { const size_t _Haystack_length_bytes = _Haystack_length * sizeof(_Ty); - constexpr int _Op = - (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_MOST_SIGNIFICANT; + constexpr int _Op_base = + (_Pred == _Find_meow_of_predicate::_Any_of ? _SIDD_POSITIVE_POLARITY : _SIDD_MASKED_NEGATIVE_POLARITY) + | (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY; + constexpr int _Op = _Op_base | _SIDD_MOST_SIGNIFICANT; constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; const size_t _Last_part_size = _Haystack_length_bytes & 0xF; @@ -3959,37 +4074,77 @@ namespace { const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); constexpr int _Not_found = -1; // equal to npos when treated as size_t; also less than any found value - int _Found_pos = _Not_found; - - const auto _Step = [&_Found_pos](const __m128i _Data2, const int _Size2, const __m128i _Data1, - const int _Size1) noexcept { - if (_mm_cmpestrc(_Data2, _Size2, _Data1, _Size1, _Op)) { - const int _Pos = _mm_cmpestri(_Data2, _Size2, _Data1, _Size1, _Op); - if (_Pos > _Found_pos) { - _Found_pos = _Pos; - } - } - }; #pragma warning(push) #pragma warning(disable : 4324) // structure was padded due to alignment specifier const auto _Test_whole_needle = [=](const __m128i _Data1, const int _Size1) noexcept { - const void* _Cur_needle = _Needle; - do { - const __m128i _Data2 = _mm_loadu_si128(static_cast(_Cur_needle)); - _Step(_Data2, _Part_size_el, _Data1, _Size1); - _Advance_bytes(_Cur_needle, 16); - } while (_Cur_needle != _Last_needle); + if constexpr (_Pred == _Find_meow_of_predicate::_Any_of) { + int _Found_pos = _Not_found; + + const auto _Step = [&_Found_pos](const __m128i _Data2, const int _Size2, const __m128i _Data1, + const int _Size1) noexcept { + if (_mm_cmpestrc(_Data2, _Size2, _Data1, _Size1, _Op)) { + const int _Pos = _mm_cmpestri(_Data2, _Size2, _Data1, _Size1, _Op); + if (_Pos > _Found_pos) { + _Found_pos = _Pos; + } + } + }; + + const void* _Cur_needle = _Needle; + do { + const __m128i _Data2 = _mm_loadu_si128(static_cast(_Cur_needle)); + _Step(_Data2, _Part_size_el, _Data1, _Size1); + _Advance_bytes(_Cur_needle, 16); + } while (_Cur_needle != _Last_needle); + + if (_Last_needle_length_el != 0) { + _Step(_Last_needle_val, _Last_needle_length_el, _Data1, _Size1); + } + + return _Found_pos; + } else { + constexpr int _Op_mask = _Op_base | _SIDD_BIT_MASK; + + const void* _Cur_needle = _Needle; + + const __m128i _Data2_first = _mm_loadu_si128(static_cast(_Cur_needle)); + + __m128i _Found = _mm_cmpestrm(_Data2_first, _Part_size_el, _Data1, _Size1, _Op_mask); + + while (_Cur_needle != _Last_needle) { + const __m128i _Data2 = _mm_loadu_si128(static_cast(_Cur_needle)); + const __m128i _Found_part = _mm_cmpestrm(_Data2, _Part_size_el, _Data1, _Size1, _Op_mask); + _Found = _mm_and_si128(_Found, _Found_part); + _Advance_bytes(_Cur_needle, 16); + } + + if (_Last_needle_length_el != 0) { + const __m128i _Found_part = + _mm_cmpestrm(_Last_needle_val, _Last_needle_length_el, _Data1, _Size1, _Op_mask); + _Found = _mm_and_si128(_Found, _Found_part); + _Advance_bytes(_Cur_needle, 16); + } + + const int _Bingo = _mm_cvtsi128_si32(_Found); + int _Found_pos = _Not_found; + + if (_Bingo != 0) { + unsigned long _Tmp; + // CodeQL [SM02313] _Tmp is always initialized: we just tested `if (_Bingo != 0)`. + _BitScanReverse(&_Tmp, _Bingo); + _Found_pos = static_cast(_Tmp); + } - if (_Last_needle_length_el != 0) { - _Step(_Last_needle_val, _Last_needle_length_el, _Data1, _Size1); + return _Found_pos; } }; #pragma warning(pop) while (_Cur != _Stop_at) { _Rewind_bytes(_Cur, 16); - _Test_whole_needle(_mm_loadu_si128(static_cast(_Cur)), _Part_size_el); + const int _Found_pos = + _Test_whole_needle(_mm_loadu_si128(static_cast(_Cur)), _Part_size_el); if (_Found_pos != _Not_found) { return _Byte_length(_Haystack, _Cur) / sizeof(_Ty) + _Found_pos; @@ -4008,15 +4163,15 @@ namespace { _Data1 = _mm_load_si128(reinterpret_cast(_Tmp1)); } - _Test_whole_needle(_Data1, _Last_part_size_el); + return _Test_whole_needle(_Data1, _Last_part_size_el); } - return static_cast(_Found_pos); + return static_cast(_Not_found); } } #endif // !_M_ARM64EC - template + template size_t _Dispatch_pos( const void* const _First1, const size_t _Count1, const void* const _First2, const size_t _Count2) noexcept { using namespace __std_find_meow_of_bitmap; @@ -4027,26 +4182,26 @@ namespace { if (_Strat == _Strategy::_Vector_bitmap) { if (_Can_fit_256_bits_sse(static_cast(_First2), _Count2)) { - return _Impl_last_avx<_Ty>(_First1, _Count1, _First2, _Count2); + return _Impl_last_avx<_Ty, _Pred>(_First1, _Count1, _First2, _Count2); } } else if (_Strat == _Strategy::_Scalar_bitmap) { if (_Can_fit_256_bits_sse(static_cast(_First2), _Count2)) { alignas(32) _Scalar_table_t _Table = {}; _Build_scalar_table_no_check<_Ty>(_First2, _Count2, _Table); - return _Impl_last_scalar<_Ty>(_First1, _Count1, _Table); + return _Impl_last_scalar<_Ty, _Pred>(_First1, _Count1, _Table); } } - return _Impl<_Ty>(_First1, _Count1, _First2, _Count2); + return _Impl<_Ty, _Pred>(_First1, _Count1, _First2, _Count2); } else #endif // !_M_ARM64EC { alignas(32) _Scalar_table_t _Table = {}; if (_Build_scalar_table<_Ty>(_First2, _Count2, _Table)) { - return _Impl_last_scalar<_Ty>(_First1, _Count1, _Table); + return _Impl_last_scalar<_Ty, _Pred>(_First1, _Count1, _Table); } - return _Fallback<_Ty>(_First1, _Count1, _First2, _Count2); + return _Fallback<_Ty, _Pred>(_First1, _Count1, _First2, _Count2); } } } // namespace __std_find_last_of @@ -4675,32 +4830,62 @@ const void* __stdcall __std_find_first_of_trivial_8( __declspec(noalias) size_t __stdcall __std_find_first_of_trivial_pos_1( const void* _Haystack, size_t _Haystack_length, const void* _Needle, size_t _Needle_length) noexcept { - return __std_find_first_of::_Dispatch_pos(_Haystack, _Haystack_length, _Needle, _Needle_length); + return __std_find_first_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); } __declspec(noalias) size_t __stdcall __std_find_first_of_trivial_pos_2( const void* _Haystack, size_t _Haystack_length, const void* _Needle, size_t _Needle_length) noexcept { - return __std_find_first_of::_Dispatch_pos(_Haystack, _Haystack_length, _Needle, _Needle_length); + return __std_find_first_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); } __declspec(noalias) size_t __stdcall __std_find_first_of_trivial_pos_4( const void* _Haystack, size_t _Haystack_length, const void* _Needle, size_t _Needle_length) noexcept { - return __std_find_first_of::_Dispatch_pos(_Haystack, _Haystack_length, _Needle, _Needle_length); + return __std_find_first_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); } __declspec(noalias) size_t __stdcall __std_find_first_of_trivial_pos_8( const void* _Haystack, size_t _Haystack_length, const void* _Needle, size_t _Needle_length) noexcept { - return __std_find_first_of::_Dispatch_pos(_Haystack, _Haystack_length, _Needle, _Needle_length); + return __std_find_first_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); } __declspec(noalias) size_t __stdcall __std_find_last_of_trivial_pos_1(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { - return __std_find_last_of::_Dispatch_pos(_Haystack, _Haystack_length, _Needle, _Needle_length); + return __std_find_last_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); } __declspec(noalias) size_t __stdcall __std_find_last_of_trivial_pos_2(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { - return __std_find_last_of::_Dispatch_pos(_Haystack, _Haystack_length, _Needle, _Needle_length); + return __std_find_last_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); +} + +__declspec(noalias) size_t __stdcall __std_find_first_not_of_trivial_pos_1(const void* const _Haystack, + const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { + return __std_find_first_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); +} + +__declspec(noalias) size_t __stdcall __std_find_first_not_of_trivial_pos_2(const void* const _Haystack, + const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { + return __std_find_first_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); +} + +__declspec(noalias) size_t __stdcall __std_find_last_not_of_trivial_pos_1(const void* const _Haystack, + const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { + return __std_find_last_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); +} + +__declspec(noalias) size_t __stdcall __std_find_last_not_of_trivial_pos_2(const void* const _Haystack, + const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { + return __std_find_last_of::_Dispatch_pos( + _Haystack, _Haystack_length, _Needle, _Needle_length); } const void* __stdcall __std_search_1( diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 9ccaadf6ea3..2e5a86f1a35 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -1064,12 +1064,25 @@ void test_bitset(mt19937_64& gen) { } template -void test_case_string_find_first_of(const basic_string& input_haystack, const basic_string& input_needle) { - auto expected_iter = last_known_good_find_first_of( - input_haystack.begin(), input_haystack.end(), input_needle.begin(), input_needle.end()); - auto expected = (expected_iter != input_haystack.end()) ? expected_iter - input_haystack.begin() : ptrdiff_t{-1}; - auto actual = static_cast(input_haystack.find_first_of(input_needle)); - assert(expected == actual); +size_t last_known_good_find_first_of(const basic_string& h, const basic_string& n) { + for (size_t pos = 0, pos_max = h.size(); pos != pos_max; ++pos) { + if (n.find(h[pos]) != basic_string::npos) { + return pos; + } + } + + return basic_string::npos; +} + +template +size_t last_known_good_find_first_not_of(const basic_string& h, const basic_string& n) { + for (size_t pos = 0, pos_max = h.size(); pos != pos_max; ++pos) { + if (n.find(h[pos]) == basic_string::npos) { + return pos; + } + } + + return basic_string::npos; } template @@ -1085,6 +1098,33 @@ size_t last_known_good_find_last_of(const basic_string& h, const basic_string return basic_string::npos; } +template +size_t last_known_good_find_last_not_of(const basic_string& h, const basic_string& n) { + size_t pos = h.size(); + while (pos != 0) { + --pos; + if (n.find(h[pos]) == basic_string::npos) { + return pos; + } + } + + return basic_string::npos; +} + +template +void test_case_string_find_first_of(const basic_string& input_haystack, const basic_string& input_needle) { + size_t expected = last_known_good_find_first_of(input_haystack, input_needle); + size_t actual = input_haystack.find_first_of(input_needle); + assert(expected == actual); +} + +template +void test_case_string_find_first_not_of(const basic_string& input_haystack, const basic_string& input_needle) { + size_t expected = last_known_good_find_first_not_of(input_haystack, input_needle); + size_t actual = input_haystack.find_first_not_of(input_needle); + assert(expected == actual); +} + template void test_case_string_find_last_of(const basic_string& input_haystack, const basic_string& input_needle) { size_t expected = last_known_good_find_last_of(input_haystack, input_needle); @@ -1092,6 +1132,13 @@ void test_case_string_find_last_of(const basic_string& input_haystack, const assert(expected == actual); } +template +void test_case_string_find_last_not_of(const basic_string& input_haystack, const basic_string& input_needle) { + size_t expected = last_known_good_find_last_not_of(input_haystack, input_needle); + size_t actual = input_haystack.find_last_not_of(input_needle); + assert(expected == actual); +} + template void test_case_string_find_ch(const basic_string& input_haystack, const T value) { ptrdiff_t expected; @@ -1216,6 +1263,8 @@ void test_basic_string_dis(mt19937_64& gen, D& dis) { test_case_string_find_first_of(input_haystack, input_needle); test_case_string_find_last_of(input_haystack, input_needle); + test_case_string_find_first_not_of(input_haystack, input_needle); + test_case_string_find_last_not_of(input_haystack, input_needle); test_case_string_find_str(input_haystack, input_needle); test_case_string_rfind_str(input_haystack, input_needle); @@ -1223,6 +1272,8 @@ void test_basic_string_dis(mt19937_64& gen, D& dis) { input_needle.push_back(static_cast(dis(gen))); test_case_string_find_first_of(input_haystack, input_needle); test_case_string_find_last_of(input_haystack, input_needle); + test_case_string_find_first_not_of(input_haystack, input_needle); + test_case_string_find_last_not_of(input_haystack, input_needle); test_case_string_find_str(input_haystack, input_needle); test_case_string_rfind_str(input_haystack, input_needle); From 5afb0328bc44f905a0024bf4fcd2f17acaf5dc47 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 24 Mar 2025 16:28:34 -0700 Subject: [PATCH 03/14] Vectorize `adjacent_find` (#5331) Co-authored-by: Stephan T. Lavavej --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/adjacent_find.cpp | 56 +++++++++ stl/inc/algorithm | 18 +++ stl/inc/xutility | 37 ++++++ stl/src/vector_algorithms.cpp | 106 ++++++++++++++++++ .../VSO_0000000_vector_algorithms/test.cpp | 71 ++++++++++++ 6 files changed, 289 insertions(+) create mode 100644 benchmarks/src/adjacent_find.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 22503345a4a..f363c012e82 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -107,6 +107,7 @@ function(add_benchmark name) endfunction() add_benchmark(adjacent_difference src/adjacent_difference.cpp) +add_benchmark(adjacent_find src/adjacent_find.cpp) add_benchmark(bitset_from_string src/bitset_from_string.cpp) add_benchmark(bitset_to_string src/bitset_to_string.cpp) add_benchmark(efficient_nonlocking_print src/efficient_nonlocking_print.cpp) diff --git a/benchmarks/src/adjacent_find.cpp b/benchmarks/src/adjacent_find.cpp new file mode 100644 index 00000000000..67036be67bf --- /dev/null +++ b/benchmarks/src/adjacent_find.cpp @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include +#include + +using namespace std; + +enum class AlgType { Std, Rng }; + +template +void bm(benchmark::State& state) { + const size_t size = static_cast(state.range(0)); + const size_t pos = static_cast(state.range(1)); + + vector v(size); + + for (size_t i = 0; i != size; ++i) { + v[i] = static_cast(i & 3); + } + + if (pos == 0 || pos >= size) { + abort(); + } + + v[pos] = v[pos - 1]; + + for (auto _ : state) { + benchmark::DoNotOptimize(v); + if constexpr (Alg == AlgType::Std) { + benchmark::DoNotOptimize(adjacent_find(v.begin(), v.end())); + } else { + benchmark::DoNotOptimize(ranges::adjacent_find(v)); + } + } +} + +void common_args(auto bm) { + bm->ArgPair(2525, 1142); +} + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK_MAIN(); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 8109baf3a16..f4f7b097c88 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -542,6 +542,24 @@ _NODISCARD _CONSTEXPR20 _FwdIt adjacent_find(const _FwdIt _First, _FwdIt _Last, auto _UFirst = _STD _Get_unwrapped(_First); auto _ULast = _STD _Get_unwrapped(_Last); if (_UFirst != _ULast) { +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (_Equal_memcmp_is_safe) { + if (!_STD _Is_constant_evaluated()) { + const auto _First_ptr = _STD _To_address(_UFirst); + const auto _Result = _STD _Adjacent_find_vectorized(_First_ptr, _STD _To_address(_ULast)); + + if constexpr (is_pointer_v) { + _ULast = _Result; + } else { + _ULast = _UFirst + (_Result - _First_ptr); + } + + _STD _Seek_wrapped(_Last, _ULast); + return _Last; + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + for (auto _UNext = _UFirst; ++_UNext != _ULast; _UFirst = _UNext) { if (_Pred(*_UFirst, *_UNext)) { _ULast = _UFirst; diff --git a/stl/inc/xutility b/stl/inc/xutility index 4abd43c637c..222054fefaf 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -98,6 +98,11 @@ const void* __stdcall __std_find_last_trivial_2(const void* _First, const void* const void* __stdcall __std_find_last_trivial_4(const void* _First, const void* _Last, uint32_t _Val) noexcept; const void* __stdcall __std_find_last_trivial_8(const void* _First, const void* _Last, uint64_t _Val) noexcept; +const void* __stdcall __std_adjacent_find_1(const void* _First, const void* _Last) noexcept; +const void* __stdcall __std_adjacent_find_2(const void* _First, const void* _Last) noexcept; +const void* __stdcall __std_adjacent_find_4(const void* _First, const void* _Last) noexcept; +const void* __stdcall __std_adjacent_find_8(const void* _First, const void* _Last) noexcept; + const void* __stdcall __std_search_1( const void* _First1, const void* _Last1, const void* _First2, size_t _Count2) noexcept; const void* __stdcall __std_search_2( @@ -240,6 +245,21 @@ _Ty* _Find_last_vectorized(_Ty* const _First, _Ty* const _Last, const _TVal _Val } } +template +_Ty* _Adjacent_find_vectorized(_Ty* const _First, _Ty* const _Last) noexcept { + if constexpr (sizeof(_Ty) == 1) { + return const_cast<_Ty*>(static_cast(::__std_adjacent_find_1(_First, _Last))); + } else if constexpr (sizeof(_Ty) == 2) { + return const_cast<_Ty*>(static_cast(::__std_adjacent_find_2(_First, _Last))); + } else if constexpr (sizeof(_Ty) == 4) { + return const_cast<_Ty*>(static_cast(::__std_adjacent_find_4(_First, _Last))); + } else if constexpr (sizeof(_Ty) == 8) { + return const_cast<_Ty*>(static_cast(::__std_adjacent_find_8(_First, _Last))); + } else { + _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size + } +} + // find_first_of vectorization is likely to be a win after this size (in elements) _INLINE_VAR constexpr ptrdiff_t _Threshold_find_first_of = 16; @@ -6786,6 +6806,23 @@ namespace ranges { return _First; } +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (_Equal_memcmp_is_safe<_It, _It, _Pr> && sized_sentinel_for<_Se, _It> + && is_same_v<_Pj, identity>) { + if (!_STD is_constant_evaluated()) { + const auto _First_ptr = _STD _To_address(_First); + const auto _Last_ptr = _First_ptr + (_Last - _First); + + const auto _Result = _STD _Adjacent_find_vectorized(_First_ptr, _Last_ptr); + if constexpr (is_pointer_v<_It>) { + return _Result; + } else { + return _First + (_Result - _First_ptr); + } + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + for (auto _Next = _First;; ++_First) { if (++_Next == _Last) { return _Next; diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 559de319996..aa7348263ee 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2751,6 +2751,96 @@ namespace { } } + template + const void* __stdcall __std_adjacent_find_impl(const void* _First, const void* const _Last) noexcept { + if (_First == _Last) { + return _Last; + } + +#ifndef _M_ARM64EC + const size_t _Size_bytes = _Byte_length(_First, _Last) - sizeof(_Ty); + + if (const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { + _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 + + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Avx_size); + + do { + const void* _Next = _First; + _Advance_bytes(_Next, sizeof(_Ty)); + + const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); + const __m256i _Comparand = _mm256_loadu_si256(static_cast(_Next)); + const int _Bingo = _mm256_movemask_epi8(_Traits::_Cmp_avx(_Data, _Comparand)); + + if (_Bingo != 0) { + const unsigned long _Offset = _tzcnt_u32(_Bingo); + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, 32); + } while (_First != _Stop_at); + + if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { + const void* _Next = _First; + _Advance_bytes(_Next, sizeof(_Ty)); + + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size >> 2); + const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); + const __m256i _Comparand = _mm256_maskload_epi32(static_cast(_Next), _Tail_mask); + const int _Bingo = + _mm256_movemask_epi8(_mm256_and_si256(_Traits::_Cmp_avx(_Data, _Comparand), _Tail_mask)); + + if (_Bingo != 0) { + const unsigned long _Offset = _tzcnt_u32(_Bingo); + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, _Avx_tail_size); + } + + if constexpr (sizeof(_Ty) >= 4) { + return _Last; + } + } else if (const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Sse_size); + + do { + const void* _Next = _First; + _Advance_bytes(_Next, sizeof(_Ty)); + + const __m128i _Data = _mm_loadu_si128(static_cast(_First)); + const __m128i _Comparand = _mm_loadu_si128(static_cast(_Next)); + const int _Bingo = _mm_movemask_epi8(_Traits::_Cmp_sse(_Data, _Comparand)); + + if (_Bingo != 0) { + unsigned long _Offset; + // CodeQL [SM02313] _Offset is always initialized: we just tested `if (_Bingo != 0)`. + _BitScanForward(&_Offset, _Bingo); + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, 16); + } while (_First != _Stop_at); + } +#endif // !_M_ARM64EC + + auto _Ptr = static_cast(_First); + auto _Next = _Ptr + 1; + for (; _Next != _Last; ++_Ptr, ++_Next) { + if (*_Ptr == *_Next) { + return _Ptr; + } + } + + return _Last; + } + struct _Count_traits_8 : _Find_traits_8 { #ifndef _M_ARM64EC static __m256i _Sub_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { @@ -4788,6 +4878,22 @@ __declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_8( return __std_find_last_pos<_Find_traits_8, _Find_one_predicate::_Not_equal>(_First, _Last, _Val); } +const void* __stdcall __std_adjacent_find_1(const void* const _First, const void* const _Last) noexcept { + return __std_adjacent_find_impl<_Find_traits_1, uint8_t>(_First, _Last); +} + +const void* __stdcall __std_adjacent_find_2(const void* const _First, const void* const _Last) noexcept { + return __std_adjacent_find_impl<_Find_traits_2, uint16_t>(_First, _Last); +} + +const void* __stdcall __std_adjacent_find_4(const void* const _First, const void* const _Last) noexcept { + return __std_adjacent_find_impl<_Find_traits_4, uint32_t>(_First, _Last); +} + +const void* __stdcall __std_adjacent_find_8(const void* const _First, const void* const _Last) noexcept { + return __std_adjacent_find_impl<_Find_traits_8, uint64_t>(_First, _Last); +} + __declspec(noalias) size_t __stdcall __std_count_trivial_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { return __std_count_trivial_impl<_Count_traits_1>(_First, _Last, _Val); diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 2e5a86f1a35..ffb9261f1a2 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -103,6 +103,67 @@ void test_adjacent_difference_with_heterogeneous_types() { assert(output == expected); } +template +FwdIt last_known_good_adj_find(FwdIt first, FwdIt last) { + if (first == last) { + return last; + } + + auto next = first; + for (++next; next != last; ++first, ++next) { + if (*first == *next) { + return first; + } + } + + return last; +} + +template +void test_case_adj_find(const vector& input) { + const auto actual = adjacent_find(input.begin(), input.end()); + const auto expected = last_known_good_adj_find(input.begin(), input.end()); + assert(actual == expected); + +#if _HAS_CXX20 + const auto actual_r = ranges::adjacent_find(input); + assert(actual_r == expected); +#endif // _HAS_CXX20 +} + +template +void test_adjacent_find(mt19937_64& gen) { + constexpr size_t replicaCount = 4; + + using Limits = numeric_limits; + + uniform_int_distribution> dis(Limits::min(), Limits::max()); + + vector original_input; + vector input; + + original_input.reserve(dataCount); + input.reserve(dataCount); + + test_case_adj_find(input); + for (size_t attempts = 0; attempts < dataCount; ++attempts) { + original_input.push_back(static_cast(dis(gen))); + input = original_input; + + test_case_adj_find(input); + + if (original_input.size() > 2) { + uniform_int_distribution pos_dis(0, original_input.size() - 2); + + for (size_t replicas = 0; replicas < replicaCount; ++replicas) { + const size_t replica_pos = pos_dis(gen); + input[replica_pos] = input[replica_pos + 1]; + test_case_adj_find(input); + } + } + } +} + template ptrdiff_t last_known_good_count(FwdIt first, FwdIt last, T v) { ptrdiff_t result = 0; @@ -763,6 +824,16 @@ void test_vector_algorithms(mt19937_64& gen) { test_adjacent_difference_with_heterogeneous_types(); + test_adjacent_find(gen); + test_adjacent_find(gen); + test_adjacent_find(gen); + test_adjacent_find(gen); + test_adjacent_find(gen); + test_adjacent_find(gen); + test_adjacent_find(gen); + test_adjacent_find(gen); + test_adjacent_find(gen); + test_count(gen); test_count(gen); test_count(gen); From 2b11023aee79119780e65c6c58bc9e6699b33a0c Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 24 Mar 2025 16:32:28 -0700 Subject: [PATCH 04/14] Vectorize `unique` (#5092) Co-authored-by: Stephan T. Lavavej --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/unique.cpp | 47 +++ stl/inc/algorithm | 61 +++ stl/src/vector_algorithms.cpp | 384 ++++++++++++++---- .../VSO_0000000_vector_algorithms/test.cpp | 85 ++++ 5 files changed, 508 insertions(+), 70 deletions(-) create mode 100644 benchmarks/src/unique.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index f363c012e82..d19deb06ebd 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -128,6 +128,7 @@ add_benchmark(search src/search.cpp) add_benchmark(std_copy src/std_copy.cpp) add_benchmark(sv_equal src/sv_equal.cpp) add_benchmark(swap_ranges src/swap_ranges.cpp) +add_benchmark(unique src/unique.cpp) add_benchmark(vector_bool_copy src/std/containers/sequences/vector.bool/copy/test.cpp) add_benchmark(vector_bool_copy_n src/std/containers/sequences/vector.bool/copy_n/test.cpp) diff --git a/benchmarks/src/unique.cpp b/benchmarks/src/unique.cpp new file mode 100644 index 00000000000..8fb20db444c --- /dev/null +++ b/benchmarks/src/unique.cpp @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include +#include + +#include "skewed_allocator.hpp" + +enum class alg_type { std_fn, rng }; + +template +void u(benchmark::State& state) { + std::mt19937_64 gen(22033); + using TD = std::conditional_t; + std::binomial_distribution dis(5); + + std::vector> src(2552); + std::generate(src.begin(), src.end(), [&] { return static_cast(dis(gen)); }); + + std::vector> v; + v.reserve(src.size()); + for (auto _ : state) { + v = src; + benchmark::DoNotOptimize(v); + if constexpr (Type == alg_type::std_fn) { + benchmark::DoNotOptimize(std::unique(v.begin(), v.end())); + } else { + benchmark::DoNotOptimize(std::ranges::unique(v)); + } + } +} + +BENCHMARK(u); +BENCHMARK(u); +BENCHMARK(u); +BENCHMARK(u); + +BENCHMARK(u); +BENCHMARK(u); +BENCHMARK(u); +BENCHMARK(u); + +BENCHMARK_MAIN(); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index f4f7b097c88..ee97590be83 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -79,6 +79,11 @@ __declspec(noalias) void __stdcall __std_replace_4( void* _First, void* _Last, uint32_t _Old_val, uint32_t _New_val) noexcept; __declspec(noalias) void __stdcall __std_replace_8( void* _First, void* _Last, uint64_t _Old_val, uint64_t _New_val) noexcept; + +void* __stdcall __std_unique_1(void* _First, void* _Last) noexcept; +void* __stdcall __std_unique_2(void* _First, void* _Last) noexcept; +void* __stdcall __std_unique_4(void* _First, void* _Last) noexcept; +void* __stdcall __std_unique_8(void* _First, void* _Last) noexcept; } // extern "C" _STD_BEGIN @@ -205,6 +210,21 @@ __declspec(noalias) void _Replace_vectorized( } } +template +_Ty* _Unique_vectorized(_Ty* const _First, _Ty* const _Last) noexcept { + if constexpr (sizeof(_Ty) == 1) { + return reinterpret_cast<_Ty*>(::__std_unique_1(_First, _Last)); + } else if constexpr (sizeof(_Ty) == 2) { + return reinterpret_cast<_Ty*>(::__std_unique_2(_First, _Last)); + } else if constexpr (sizeof(_Ty) == 4) { + return reinterpret_cast<_Ty*>(::__std_unique_4(_First, _Last)); + } else if constexpr (sizeof(_Ty) == 8) { + return reinterpret_cast<_Ty*>(::__std_unique_8(_First, _Last)); + } else { + _STL_INTERNAL_STATIC_ASSERT(false); // Unexpected size + } +} + // Can we activate the vector algorithms for find_first_of? template constexpr bool _Vector_alg_in_find_first_of_is_safe = _Equal_memcmp_is_safe<_It1, _It2, _Pr>; @@ -219,6 +239,10 @@ template constexpr bool _Vector_alg_in_ranges_replace_is_safe = _Vector_alg_in_replace_is_safe<_Iter, _Ty1> // can search and replace && _Vector_alg_in_find_is_safe_elem<_Ty2, _Iter_value_t<_Iter>>; // replacement fits + +// Can we activate the vector algorithms for unique? +template +constexpr bool _Vector_alg_in_unique_is_safe = _Equal_memcmp_is_safe<_Iter, _Iter, _Pr>; _STD_END #endif // _USE_STD_VECTOR_ALGORITHMS @@ -4869,6 +4893,25 @@ _NODISCARD_UNIQUE_ALG _CONSTEXPR20 _FwdIt unique(_FwdIt _First, _FwdIt _Last, _P _STD _Adl_verify_range(_First, _Last); auto _UFirst = _STD _Get_unwrapped(_First); const auto _ULast = _STD _Get_unwrapped(_Last); + +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (_Vector_alg_in_unique_is_safe) { + if (!_STD _Is_constant_evaluated()) { + const auto _First_ptr = _STD _To_address(_UFirst); + const auto _Result = _STD _Unique_vectorized(_First_ptr, _STD _To_address(_ULast)); + + if constexpr (is_pointer_v) { + _UFirst = _Result; + } else { + _UFirst += _Result - _First_ptr; + } + + _STD _Seek_wrapped(_Last, _UFirst); + return _Last; + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + if (_UFirst != _ULast) { for (auto _UFirstb = _UFirst; ++_UFirst != _ULast; _UFirstb = _UFirst) { if (_Pred(*_UFirstb, *_UFirst)) { // copy down @@ -4945,6 +4988,24 @@ namespace ranges { _STL_INTERNAL_STATIC_ASSERT(sentinel_for<_Se, _It>); _STL_INTERNAL_STATIC_ASSERT(indirect_equivalence_relation<_Pr, projected<_It, _Pj>>); +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (is_same_v<_Pj, identity> && sized_sentinel_for<_Se, _It> + && _Vector_alg_in_unique_is_safe<_It, _Pr>) { + if (!_STD is_constant_evaluated()) { + const auto _Size = _Last - _First; + const auto _First_ptr = _STD to_address(_First); + const auto _Last_ptr = _First_ptr + static_cast(_Size); + const auto _Result = _STD _Unique_vectorized(_First_ptr, _Last_ptr); + + if constexpr (is_pointer_v<_It>) { + return {_Result, _Last_ptr}; + } else { + return {_First + (_Result - _First_ptr), _First + _Size}; + } + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + auto _Current = _First; if (_First == _Last) { return {_STD move(_Current), _STD move(_First)}; diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index aa7348263ee..a267d94ca36 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -5135,6 +5135,24 @@ namespace { return _Dest; } + template + void* _Unique_fallback(void* const _First, void* const _Last, void* const _Dest) noexcept { + _Ty* _Out = reinterpret_cast<_Ty*>(_Dest); + _Ty* _Src = reinterpret_cast<_Ty*>(_First); + + while (_Src != _Last) { + if (*_Src != *_Out) { + ++_Out; + *_Out = *_Src; + } + + ++_Src; + } + + ++_Out; + return _Out; + } + #ifndef _M_ARM64EC template struct _Remove_tables { @@ -5169,6 +5187,8 @@ namespace { // It is not possible to leave them untouched while keeping this optimization efficient. // This should not be a problem though, as they should be either overwritten by the next step, // or left in the removed range. + // 'remove' does not require any specific values, + // 'unique' needs the last element value to be preserved, as it will be loaded again. for (; _Nx != _Size_h / _Ew; ++_Nx) { // Inner loop needed for cases where the shuffle mask operates on element parts rather than whole // elements; for whole elements there would be one iteration. @@ -5187,6 +5207,195 @@ namespace { constexpr auto _Remove_tables_4_avx = _Make_remove_tables<256, 8>(4, 1); constexpr auto _Remove_tables_8_sse = _Make_remove_tables<4, 16>(8, 8); constexpr auto _Remove_tables_8_avx = _Make_remove_tables<16, 8>(8, 2); + + struct _Remove_sse_1 { + static constexpr size_t _Elem_size = 1; + static constexpr size_t _Step = 8; + + static __m128i _Set(const uint8_t _Val) noexcept { + return _mm_shuffle_epi8(_mm_cvtsi32_si128(_Val), _mm_setzero_si128()); + } + + static __m128i _Load(const void* const _Ptr) noexcept { + return _mm_loadu_si64(_Ptr); + } + + static uint32_t _Mask(const __m128i _First, const __m128i _Second) noexcept { + return _mm_movemask_epi8(_mm_cmpeq_epi8(_First, _Second)) & 0xFF; + } + + static void* _Store_masked(void* _Out, const __m128i _Src, const uint32_t _Bingo) noexcept { + const __m128i _Shuf = _mm_loadu_si64(_Remove_tables_1_sse._Shuf[_Bingo]); + const __m128i _Dest = _mm_shuffle_epi8(_Src, _Shuf); + _mm_storeu_si64(_Out, _Dest); + _Advance_bytes(_Out, _Remove_tables_1_sse._Size[_Bingo]); + return _Out; + } + }; + + struct _Remove_sse_2 { + static constexpr size_t _Elem_size = 2; + static constexpr size_t _Step = 16; + + static __m128i _Set(const uint16_t _Val) noexcept { + return _mm_set1_epi16(_Val); + } + + static __m128i _Load(const void* const _Ptr) noexcept { + return _mm_loadu_si128(reinterpret_cast(_Ptr)); + } + + static uint32_t _Mask(const __m128i _First, const __m128i _Second) noexcept { + const __m128i _Mask = _mm_cmpeq_epi16(_First, _Second); + return _mm_movemask_epi8(_mm_packs_epi16(_Mask, _mm_setzero_si128())); + } + + static void* _Store_masked(void* _Out, const __m128i _Src, const uint32_t _Bingo) noexcept { + const __m128i _Shuf = _mm_loadu_si128(reinterpret_cast(_Remove_tables_2_sse._Shuf[_Bingo])); + const __m128i _Dest = _mm_shuffle_epi8(_Src, _Shuf); + _mm_storeu_si128(reinterpret_cast<__m128i*>(_Out), _Dest); + _Advance_bytes(_Out, _Remove_tables_2_sse._Size[_Bingo]); + return _Out; + } + }; + + struct _Remove_avx_4 { + static constexpr size_t _Elem_size = 4; + static constexpr size_t _Step = 32; + + static __m256i _Set(const uint32_t _Val) noexcept { + return _mm256_set1_epi32(_Val); + } + + static __m256i _Load(const void* const _Ptr) noexcept { + return _mm256_loadu_si256(reinterpret_cast(_Ptr)); + } + + static uint32_t _Mask(const __m256i _First, const __m256i _Second) noexcept { + const __m256i _Mask = _mm256_cmpeq_epi32(_First, _Second); + return _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)); + } + + static void* _Store_masked(void* _Out, const __m256i _Src, const uint32_t _Bingo) noexcept { + const __m256i _Shuf = _mm256_cvtepu8_epi32(_mm_loadu_si64(_Remove_tables_4_avx._Shuf[_Bingo])); + const __m256i _Dest = _mm256_permutevar8x32_epi32(_Src, _Shuf); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(_Out), _Dest); + _Advance_bytes(_Out, _Remove_tables_4_avx._Size[_Bingo]); + return _Out; + } + }; + + struct _Remove_sse_4 { + static constexpr size_t _Elem_size = 4; + static constexpr size_t _Step = 16; + + static __m128i _Set(const uint32_t _Val) noexcept { + return _mm_set1_epi32(_Val); + } + + static __m128i _Load(const void* const _Ptr) noexcept { + return _mm_loadu_si128(reinterpret_cast(_Ptr)); + } + + static uint32_t _Mask(const __m128i _First, const __m128i _Second) noexcept { + const __m128i _Mask = _mm_cmpeq_epi32(_First, _Second); + return _mm_movemask_ps(_mm_castsi128_ps(_Mask)); + } + + static void* _Store_masked(void* _Out, const __m128i _Src, const uint32_t _Bingo) noexcept { + const __m128i _Shuf = _mm_loadu_si128(reinterpret_cast(_Remove_tables_4_sse._Shuf[_Bingo])); + const __m128i _Dest = _mm_shuffle_epi8(_Src, _Shuf); + _mm_storeu_si128(reinterpret_cast<__m128i*>(_Out), _Dest); + _Advance_bytes(_Out, _Remove_tables_4_sse._Size[_Bingo]); + return _Out; + } + }; + + struct _Remove_avx_8 { + static constexpr size_t _Elem_size = 8; + static constexpr size_t _Step = 32; + + static __m256i _Set(const uint64_t _Val) noexcept { + return _mm256_set1_epi64x(_Val); + } + + static __m256i _Load(const void* const _Ptr) noexcept { + return _mm256_loadu_si256(reinterpret_cast(_Ptr)); + } + + static uint32_t _Mask(const __m256i _First, const __m256i _Second) noexcept { + const __m256i _Mask = _mm256_cmpeq_epi64(_First, _Second); + return _mm256_movemask_pd(_mm256_castsi256_pd(_Mask)); + } + + static void* _Store_masked(void* _Out, const __m256i _Src, const uint32_t _Bingo) noexcept { + const __m256i _Shuf = _mm256_cvtepu8_epi32(_mm_loadu_si64(_Remove_tables_8_avx._Shuf[_Bingo])); + const __m256i _Dest = _mm256_permutevar8x32_epi32(_Src, _Shuf); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(_Out), _Dest); + _Advance_bytes(_Out, _Remove_tables_8_avx._Size[_Bingo]); + return _Out; + } + }; + + struct _Remove_sse_8 { + static constexpr size_t _Elem_size = 8; + static constexpr size_t _Step = 16; + + static __m128i _Set(const uint64_t _Val) noexcept { + return _mm_set1_epi64x(_Val); + } + + static __m128i _Load(const void* const _Ptr) noexcept { + return _mm_loadu_si128(reinterpret_cast(_Ptr)); + } + + static uint32_t _Mask(const __m128i _First, const __m128i _Second) noexcept { + const __m128i _Mask = _mm_cmpeq_epi64(_First, _Second); + return _mm_movemask_pd(_mm_castsi128_pd(_Mask)); + } + + static void* _Store_masked(void* _Out, const __m128i _Src, const uint32_t _Bingo) noexcept { + const __m128i _Shuf = _mm_loadu_si128(reinterpret_cast(_Remove_tables_8_sse._Shuf[_Bingo])); + const __m128i _Dest = _mm_shuffle_epi8(_Src, _Shuf); + _mm_storeu_si128(reinterpret_cast<__m128i*>(_Out), _Dest); + _Advance_bytes(_Out, _Remove_tables_8_sse._Size[_Bingo]); + return _Out; + } + }; + + template + void* _Remove_impl(void* _First, const void* _Stop, const _Ty _Val) noexcept { + void* _Out = _First; + const auto _Match = _Traits::_Set(_Val); + + do { + const auto _Src = _Traits::_Load(_First); + const uint32_t _Bingo = _Traits::_Mask(_Src, _Match); + _Out = _Traits::_Store_masked(_Out, _Src, _Bingo); + _Advance_bytes(_First, _Traits::_Step); + } while (_First != _Stop); + + return _Out; + } + + template + void* _Unique_impl(void* _First, const void* _Stop) noexcept { + void* _Out = _First; + + do { + const auto _Src = _Traits::_Load(_First); + void* _First_d = _First; + _Rewind_bytes(_First_d, _Traits::_Elem_size); + const auto _Match = _Traits::_Load(_First_d); + const uint32_t _Bingo = _Traits::_Mask(_Src, _Match); + _Out = _Traits::_Store_masked(_Out, _Src, _Bingo); + _Advance_bytes(_First, _Traits::_Step); + } while (_First != _Stop); + + _Rewind_bytes(_Out, _Traits::_Elem_size); + return _Out; + } + #endif // !defined(_M_ARM64EC) } // unnamed namespace @@ -5197,19 +5406,10 @@ void* __stdcall __std_remove_1(void* _First, void* const _Last, const uint8_t _V #ifndef _M_ARM64EC if (const size_t _Size_bytes = _Byte_length(_First, _Last); _Use_sse42() && _Size_bytes >= 8) { - const __m128i _Match = _mm_shuffle_epi8(_mm_cvtsi32_si128(_Val), _mm_setzero_si128()); - void* _Stop = _First; _Advance_bytes(_Stop, _Size_bytes & ~size_t{7}); - do { - const __m128i _Src = _mm_loadu_si64(_First); - const uint32_t _Bingo = _mm_movemask_epi8(_mm_cmpeq_epi8(_Src, _Match)) & 0xFF; - const __m128i _Shuf = _mm_loadu_si64(_Remove_tables_1_sse._Shuf[_Bingo]); - const __m128i _Dest = _mm_shuffle_epi8(_Src, _Shuf); - _mm_storeu_si64(_Out, _Dest); - _Advance_bytes(_Out, _Remove_tables_1_sse._Size[_Bingo]); - _Advance_bytes(_First, 8); - } while (_First != _Stop); + _Out = _Remove_impl<_Remove_sse_1>(_First, _Stop, _Val); + _First = _Stop; } #endif // !defined(_M_ARM64EC) @@ -5221,20 +5421,10 @@ void* __stdcall __std_remove_2(void* _First, void* const _Last, const uint16_t _ #ifndef _M_ARM64EC if (const size_t _Size_bytes = _Byte_length(_First, _Last); _Use_sse42() && _Size_bytes >= 16) { - const __m128i _Match = _mm_set1_epi16(_Val); - void* _Stop = _First; _Advance_bytes(_Stop, _Size_bytes & ~size_t{0xF}); - do { - const __m128i _Src = _mm_loadu_si128(reinterpret_cast(_First)); - const __m128i _Mask = _mm_cmpeq_epi16(_Src, _Match); - const uint32_t _Bingo = _mm_movemask_epi8(_mm_packs_epi16(_Mask, _mm_setzero_si128())); - const __m128i _Shuf = _mm_loadu_si128(reinterpret_cast(_Remove_tables_2_sse._Shuf[_Bingo])); - const __m128i _Dest = _mm_shuffle_epi8(_Src, _Shuf); - _mm_storeu_si128(reinterpret_cast<__m128i*>(_Out), _Dest); - _Advance_bytes(_Out, _Remove_tables_2_sse._Size[_Bingo]); - _Advance_bytes(_First, 16); - } while (_First != _Stop); + _Out = _Remove_impl<_Remove_sse_2>(_First, _Stop, _Val); + _First = _Stop; } #endif // !defined(_M_ARM64EC) @@ -5246,37 +5436,17 @@ void* __stdcall __std_remove_4(void* _First, void* const _Last, const uint32_t _ #ifndef _M_ARM64EC if (const size_t _Size_bytes = _Byte_length(_First, _Last); _Use_avx2() && _Size_bytes >= 32) { - const __m256i _Match = _mm256_set1_epi32(_Val); - void* _Stop = _First; _Advance_bytes(_Stop, _Size_bytes & ~size_t{0x1F}); - do { - const __m256i _Src = _mm256_loadu_si256(reinterpret_cast(_First)); - const __m256i _Mask = _mm256_cmpeq_epi32(_Src, _Match); - const uint32_t _Bingo = _mm256_movemask_ps(_mm256_castsi256_ps(_Mask)); - const __m256i _Shuf = _mm256_cvtepu8_epi32(_mm_loadu_si64(_Remove_tables_4_avx._Shuf[_Bingo])); - const __m256i _Dest = _mm256_permutevar8x32_epi32(_Src, _Shuf); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(_Out), _Dest); - _Advance_bytes(_Out, _Remove_tables_4_avx._Size[_Bingo]); - _Advance_bytes(_First, 32); - } while (_First != _Stop); + _Out = _Remove_impl<_Remove_avx_4>(_First, _Stop, _Val); + _First = _Stop; _mm256_zeroupper(); // TRANSITION, DevCom-10331414 } else if (_Use_sse42() && _Size_bytes >= 16) { - const __m128i _Match = _mm_set1_epi32(_Val); - void* _Stop = _First; _Advance_bytes(_Stop, _Size_bytes & ~size_t{0xF}); - do { - const __m128i _Src = _mm_loadu_si128(reinterpret_cast(_First)); - const __m128i _Mask = _mm_cmpeq_epi32(_Src, _Match); - const uint32_t _Bingo = _mm_movemask_ps(_mm_castsi128_ps(_Mask)); - const __m128i _Shuf = _mm_loadu_si128(reinterpret_cast(_Remove_tables_4_sse._Shuf[_Bingo])); - const __m128i _Dest = _mm_shuffle_epi8(_Src, _Shuf); - _mm_storeu_si128(reinterpret_cast<__m128i*>(_Out), _Dest); - _Advance_bytes(_Out, _Remove_tables_4_sse._Size[_Bingo]); - _Advance_bytes(_First, 16); - } while (_First != _Stop); + _Out = _Remove_impl<_Remove_sse_4>(_First, _Stop, _Val); + _First = _Stop; } #endif // !defined(_M_ARM64EC) @@ -5288,41 +5458,115 @@ void* __stdcall __std_remove_8(void* _First, void* const _Last, const uint64_t _ #ifndef _M_ARM64EC if (const size_t _Size_bytes = _Byte_length(_First, _Last); _Use_avx2() && _Size_bytes >= 32) { - const __m256i _Match = _mm256_set1_epi64x(_Val); + void* _Stop = _First; + _Advance_bytes(_Stop, _Size_bytes & ~size_t{0x1F}); + _Out = _Remove_impl<_Remove_avx_8>(_First, _Stop, _Val); + _First = _Stop; + + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + } else if (_Use_sse42() && _Size_bytes >= 16) { + void* _Stop = _First; + _Advance_bytes(_Stop, _Size_bytes & ~size_t{0xF}); + _Out = _Remove_impl<_Remove_sse_8>(_First, _Stop, _Val); + _First = _Stop; + } +#endif // !defined(_M_ARM64EC) + return _Remove_fallback(_First, _Last, _Out, _Val); +} + +void* __stdcall __std_unique_1(void* _First, void* _Last) noexcept { + if (_First == _Last) { + return _First; + } + + void* _Dest = _First; + _Advance_bytes(_First, 1); + +#ifndef _M_ARM64EC + if (const size_t _Size_bytes = _Byte_length(_First, _Last); _Use_sse42() && _Size_bytes >= 8) { + void* _Stop = _First; + _Advance_bytes(_Stop, _Size_bytes & ~size_t{7}); + _Dest = _Unique_impl<_Remove_sse_1>(_First, _Stop); + _First = _Stop; + } +#endif // !defined(_M_ARM64EC) + + return _Unique_fallback(_First, _Last, _Dest); +} + +void* __stdcall __std_unique_2(void* _First, void* _Last) noexcept { + if (_First == _Last) { + return _First; + } + + void* _Dest = _First; + _Advance_bytes(_First, 2); + +#ifndef _M_ARM64EC + if (const size_t _Size_bytes = _Byte_length(_First, _Last); _Use_sse42() && _Size_bytes >= 16) { + void* _Stop = _First; + _Advance_bytes(_Stop, _Size_bytes & ~size_t{0xF}); + _Dest = _Unique_impl<_Remove_sse_2>(_First, _Stop); + _First = _Stop; + } +#endif // !defined(_M_ARM64EC) + + return _Unique_fallback(_First, _Last, _Dest); +} + +void* __stdcall __std_unique_4(void* _First, void* _Last) noexcept { + if (_First == _Last) { + return _First; + } + + void* _Dest = _First; + _Advance_bytes(_First, 4); + +#ifndef _M_ARM64EC + if (const size_t _Size_bytes = _Byte_length(_First, _Last); _Use_avx2() && _Size_bytes >= 32) { void* _Stop = _First; _Advance_bytes(_Stop, _Size_bytes & ~size_t{0x1F}); - do { - const __m256i _Src = _mm256_loadu_si256(reinterpret_cast(_First)); - const __m256i _Mask = _mm256_cmpeq_epi64(_Src, _Match); - const uint32_t _Bingo = _mm256_movemask_pd(_mm256_castsi256_pd(_Mask)); - const __m256i _Shuf = _mm256_cvtepu8_epi32(_mm_loadu_si64(_Remove_tables_8_avx._Shuf[_Bingo])); - const __m256i _Dest = _mm256_permutevar8x32_epi32(_Src, _Shuf); - _mm256_storeu_si256(reinterpret_cast<__m256i*>(_Out), _Dest); - _Advance_bytes(_Out, _Remove_tables_8_avx._Size[_Bingo]); - _Advance_bytes(_First, 32); - } while (_First != _Stop); + _Dest = _Unique_impl<_Remove_avx_4>(_First, _Stop); + _First = _Stop; _mm256_zeroupper(); // TRANSITION, DevCom-10331414 } else if (_Use_sse42() && _Size_bytes >= 16) { - const __m128i _Match = _mm_set1_epi64x(_Val); + void* _Stop = _First; + _Advance_bytes(_Stop, _Size_bytes & ~size_t{0xF}); + _Dest = _Unique_impl<_Remove_sse_4>(_First, _Stop); + _First = _Stop; + } +#endif // !defined(_M_ARM64EC) + + return _Unique_fallback(_First, _Last, _Dest); +} +void* __stdcall __std_unique_8(void* _First, void* _Last) noexcept { + if (_First == _Last) { + return _First; + } + + void* _Dest = _First; + _Advance_bytes(_First, 8); + +#ifndef _M_ARM64EC + if (const size_t _Size_bytes = _Byte_length(_First, _Last); _Use_avx2() && _Size_bytes >= 32) { + void* _Stop = _First; + _Advance_bytes(_Stop, _Size_bytes & ~size_t{0x1F}); + _Dest = _Unique_impl<_Remove_avx_8>(_First, _Stop); + _First = _Stop; + + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + } else if (_Use_sse42() && _Size_bytes >= 16) { void* _Stop = _First; _Advance_bytes(_Stop, _Size_bytes & ~size_t{0xF}); - do { - const __m128i _Src = _mm_loadu_si128(reinterpret_cast(_First)); - const __m128i _Mask = _mm_cmpeq_epi64(_Src, _Match); - const uint32_t _Bingo = _mm_movemask_pd(_mm_castsi128_pd(_Mask)); - const __m128i _Shuf = _mm_loadu_si128(reinterpret_cast(_Remove_tables_8_sse._Shuf[_Bingo])); - const __m128i _Dest = _mm_shuffle_epi8(_Src, _Shuf); - _mm_storeu_si128(reinterpret_cast<__m128i*>(_Out), _Dest); - _Advance_bytes(_Out, _Remove_tables_8_sse._Size[_Bingo]); - _Advance_bytes(_First, 16); - } while (_First != _Stop); + _Dest = _Unique_impl<_Remove_sse_8>(_First, _Stop); + _First = _Stop; } #endif // !defined(_M_ARM64EC) - return _Remove_fallback(_First, _Last, _Out, _Val); + return _Unique_fallback(_First, _Last, _Dest); } } // extern "C" diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index ffb9261f1a2..13134bdd862 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include +#include #include #include #include @@ -759,6 +760,78 @@ void test_remove(mt19937_64& gen) { } } +template +FwdIt last_known_good_unique(FwdIt first, FwdIt last) { + if (first == last) { + return first; + } + + FwdIt dest = first; + ++first; + + while (first != last) { + if (*first != *dest) { + ++dest; + *dest = *first; + } + + ++first; + } + + ++dest; + return dest; +} + +template +void test_case_unique(vector& in_out_expected, vector& in_out_actual, vector& in_out_actual_r) { + auto un_expected = last_known_good_unique(in_out_expected.begin(), in_out_expected.end()); + auto un_actual = unique(in_out_actual.begin(), in_out_actual.end()); + assert(equal(in_out_expected.begin(), un_expected, in_out_actual.begin(), un_actual)); + +#if _HAS_CXX20 + auto un_actual_r = ranges::unique(in_out_actual_r); + assert(equal(in_out_expected.begin(), un_expected, begin(in_out_actual_r), begin(un_actual_r))); +#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv + (void) in_out_actual_r; +#endif // ^^^ !_HAS_CXX20 ^^^ +} + +template +void test_unique(mt19937_64& gen) { + constexpr int number_of_values = 5; + + struct unused_t {}; + + conditional_t, array, number_of_values>, unused_t> ptr_val_array{}; + + using TD = conditional_t, int, T>; + binomial_distribution dis(number_of_values); + + vector source; + vector in_out_expected; + vector in_out_actual; + vector in_out_actual_r; + + for (const auto& v : {&source, &in_out_expected, &in_out_actual, &in_out_actual_r}) { + v->reserve(dataCount); + } + + test_case_unique(in_out_expected, in_out_actual, in_out_actual_r); + for (size_t attempts = 0; attempts < dataCount; ++attempts) { + if constexpr (is_pointer_v) { + source.push_back(ptr_val_array.data() + dis(gen)); + } else { + source.push_back(static_cast(dis(gen))); + } + + for (const auto& v : {&in_out_expected, &in_out_actual, &in_out_actual_r}) { + *v = source; + } + + test_case_unique(in_out_expected, in_out_actual, in_out_actual_r); + } +} + template void test_swap_ranges(mt19937_64& gen) { const auto fn = [&]() { return static_cast(gen()); }; @@ -963,6 +1036,18 @@ void test_vector_algorithms(mt19937_64& gen) { test_remove(gen); test_remove(gen); + test_unique(gen); + test_unique(gen); + test_unique(gen); + test_unique(gen); + test_unique(gen); + test_unique(gen); + test_unique(gen); + test_unique(gen); + test_unique(gen); + + test_unique(gen); + test_swap_ranges(gen); test_swap_ranges(gen); test_swap_ranges(gen); From 7b38f9fc067b83c6dee2d30f6f631a61853c6e97 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 24 Mar 2025 16:36:11 -0700 Subject: [PATCH 05/14] ``: avoid `continue` in `ranges::search_n` (#5343) --- stl/inc/algorithm | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index ee97590be83..3399f3b5918 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -2426,7 +2426,11 @@ namespace ranges { // |=======|????????|========|??????... --_Mid2; - if (!_STD invoke(_Pred, _STD invoke(_Proj, *_Mid2), _Val)) { // mismatch; skip past it + if (_STD invoke(_Pred, _STD invoke(_Proj, *_Mid2), _Val)) { + if (_Mid2 == _Mid1) { // [_Mid1, _Mid2) is empty, so [_First, _Last) all match + return {_STD move(_First), _STD move(_Last)}; + } + } else { // mismatch; skip past it ++_Mid2; const auto _Delta = _RANGES distance(_First, _Mid2); @@ -2441,11 +2445,6 @@ namespace ranges { _Mid1 = _Last; _RANGES advance(_Last, _Delta); _Mid2 = _Last; - continue; - } - - if (_Mid2 == _Mid1) { // [_Mid1, _Mid2) is empty, so [_First, _Last) all match - return {_STD move(_First), _STD move(_Last)}; } } } else { From 0a0514cb0ecc7dd7803c7ee3685e7a0cd0d5a15c Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 24 Mar 2025 16:38:29 -0700 Subject: [PATCH 06/14] Use `find` for `search_n` when n=1 (#5346) Co-authored-by: Stephan T. Lavavej --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/search_n.cpp | 56 +++++++++++++++++++ stl/inc/algorithm | 37 ++++++++++++ .../P0896R4_ranges_alg_search_n/test.cpp | 28 ++++++++++ 4 files changed, 122 insertions(+) create mode 100644 benchmarks/src/search_n.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index d19deb06ebd..b551efa42cc 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -125,6 +125,7 @@ add_benchmark(random_integer_generation src/random_integer_generation.cpp) add_benchmark(remove src/remove.cpp) add_benchmark(replace src/replace.cpp) add_benchmark(search src/search.cpp) +add_benchmark(search_n src/search_n.cpp) add_benchmark(std_copy src/std_copy.cpp) add_benchmark(sv_equal src/sv_equal.cpp) add_benchmark(swap_ranges src/swap_ranges.cpp) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp new file mode 100644 index 00000000000..b35cfd9eb2d --- /dev/null +++ b/benchmarks/src/search_n.cpp @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include + +#include "skewed_allocator.hpp" + +using namespace std; + +// NB: This particular algorithm has std and ranges implementations with different perf characteristics! + +enum class AlgType { Std, Rng }; + +template +void bm(benchmark::State& state) { + const auto size = static_cast(state.range(0)); + + constexpr size_t N = 1; + + constexpr T no_match{'-'}; + constexpr T match{'*'}; + + vector> v(size, no_match); + + fill(v.begin() + v.size() / 2, v.end(), match); + + for (auto _ : state) { + if constexpr (Alg == AlgType::Std) { + benchmark::DoNotOptimize(search_n(v.begin(), v.end(), N, match)); + } else if constexpr (Alg == AlgType::Rng) { + benchmark::DoNotOptimize(ranges::search_n(v, N, match)); + } + } +} + +void common_args(auto bm) { + bm->Arg(3000); +} + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK_MAIN(); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 3399f3b5918..04a8a9d7496 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -2262,6 +2262,16 @@ _NODISCARD _CONSTEXPR20 _FwdIt search_n( return _First; } + if constexpr (_Is_any_of_v<_Pr, +#if _HAS_CXX20 + _RANGES equal_to, +#endif + equal_to<>>) { + if (_Count == 1) { + return _STD find(_First, _Last, _Val); + } + } + if (static_cast(_Count) > static_cast(_STD _Max_limit<_Iter_diff_t<_FwdIt>>())) { // if the number of _Vals searched for is larger than the longest possible sequence, we can't find it return _Last; @@ -2362,6 +2372,19 @@ namespace ranges { return {_First, _First}; } + if constexpr (_Is_any_of_v<_Pr, _STD equal_to<>, _RANGES equal_to>) { + if (_Count == 1) { + auto _Res = _RANGES find(_First, _Last, _Val, _Pass_fn(_Proj)); + if (_Res != _Last) { + auto _Res_end = _Res; + ++_Res_end; + return {_STD move(_Res), _STD move(_Res_end)}; + } else { + return {_Res, _Res}; + } + } + } + auto _UFirst = _RANGES _Unwrap_iter<_Se>(_STD move(_First)); auto _ULast = _RANGES _Unwrap_sent<_It>(_STD move(_Last)); @@ -2388,6 +2411,20 @@ namespace ranges { return {_First, _First}; } + if constexpr (_Is_any_of_v<_Pr, _STD equal_to<>, _RANGES equal_to>) { + if (_Count == 1) { + auto _Res = _RANGES find(_Range, _Val, _Pass_fn(_Proj)); + auto _Last = _RANGES end(_Range); + if (_Res != _Last) { + auto _Res_end = _Res; + ++_Res_end; + return {_STD move(_Res), _STD move(_Res_end)}; + } else { + return {_Res, _Res}; + } + } + } + if constexpr (sized_range<_Rng>) { const auto _Dist = _RANGES distance(_Range); diff --git a/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp b/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp index 3f3a609e568..46a2590bdb7 100644 --- a/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp +++ b/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp @@ -98,6 +98,34 @@ struct instantiator { assert(result.end() == range.begin()); } + // trivial case: unit needle + { + const auto result = ranges::search_n(range, 1, P{1, 42}); + static_assert(same_as>>); + assert(result.begin() == ranges::next(range.begin(), 1)); + assert(result.end() == ranges::next(range.begin(), 2)); + } + { + const auto result = ranges::search_n(ranges::begin(range), ranges::end(range), 1, P{1, 42}); + static_assert(same_as>>); + assert(result.begin() == ranges::next(range.begin(), 1)); + assert(result.end() == ranges::next(range.begin(), 2)); + } + + // trivial case: unit needle with predicate + { + const auto result = ranges::search_n(range, 1, 0, cmp, get_first); + static_assert(same_as>>); + assert(result.begin() == ranges::next(range.begin(), 1)); + assert(result.end() == ranges::next(range.begin(), 2)); + } + { + const auto result = ranges::search_n(ranges::begin(range), ranges::end(range), 1, 0, cmp, get_first); + static_assert(same_as>>); + assert(result.begin() == ranges::next(range.begin(), 1)); + assert(result.end() == ranges::next(range.begin(), 2)); + } + // trivial case: range too small { const auto result = ranges::search_n(range, 99999, 0, cmp, get_first); From 79c5fba7d2c05a0684ad045eae32e85edc30bd0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Tue, 25 Mar 2025 00:44:00 +0100 Subject: [PATCH 07/14] ``: Revise caret parsing in basic and grep mode (#5165) Co-authored-by: Stephan T. Lavavej --- stl/inc/regex | 40 +-- .../std/tests/VSO_0000000_regex_use/test.cpp | 326 ++++++++++++++++++ 2 files changed, 342 insertions(+), 24 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index 8057d32946a..0d1d5b84f5d 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -1511,7 +1511,6 @@ public: using _Difft = typename iterator_traits<_FwdIt>::difference_type; _Builder(const _RxTraits& _Tr, regex_constants::syntax_option_type); - bool _Beg_expr() const; void _Setlong(); // _Discard_pattern is an ABI zombie name void _Tidy() noexcept; @@ -1547,7 +1546,6 @@ private: static void _Insert_node(_Node_base*, _Node_base*); _Node_base* _New_node(_Node_type _Kind); void _Add_str_node(); - bool _Beg_expr(_Node_base*) const; void _Add_char_to_bitmap(_Elem _Ch); void _Add_char_to_array(_Elem _Ch); void _Add_elts(_Node_class<_Elem, _RxTraits>*, typename _RxTraits::char_class_type, bool); @@ -2784,17 +2782,6 @@ _Node_base* _Builder<_FwdIt, _Elem, _RxTraits>::_Getmark() const { return _Current; } -template -bool _Builder<_FwdIt, _Elem, _RxTraits>::_Beg_expr(_Node_base* _Nx) const { - // test for beginning of expression or subexpression - return _Nx->_Kind == _N_begin || _Nx->_Kind == _N_group || _Nx->_Kind == _N_capture; -} - -template -bool _Builder<_FwdIt, _Elem, _RxTraits>::_Beg_expr() const { // test for beginning of expression or subexpression - return _Beg_expr(_Current) || (_Current->_Kind == _N_bol && _Beg_expr(_Current->_Prev)); -} - template _Node_base* _Builder<_FwdIt, _Elem, _RxTraits>::_Link_node(_Node_base* _Nx) { // insert _Nx at current location _Nx->_Prev = _Current; @@ -3905,17 +3892,16 @@ void _Parser<_FwdIt, _Elem, _RxTraits>::_Trans() { // map character to meta-char break; case _Meta_star: - if ((_L_flags & _L_star_beg) && _Nfa._Beg_expr()) { - _Mchar = _Meta_chr; - } - + // A star can always act as a quantifier outside bracket expressions, + // but _L_star_beg (used by basic/grep) allows its use as an ordinary character + // at the beginning of a (sub-)expression (potentially after an optional caret anchor). + // We'll handle that when we are parsing alternatives in disjunctions. break; case _Meta_caret: - if ((_L_flags & _L_anch_rstr) && !_Nfa._Beg_expr()) { - _Mchar = _Meta_chr; - } - + // A caret can always negate a bracket expression, + // but _L_anch_rstr (used by basic/grep) restricts caret anchors to the beginning. + // We'll handle that restriction when we're about to add a bol node. break; case _Meta_dlr: @@ -4481,15 +4467,21 @@ bool _Parser<_FwdIt, _Elem, _RxTraits>::_Alternative() { // check for valid alte _Next(); _Quant = _Wrapped_disjunction(); _Expect(_Meta_rpar, regex_constants::error_paren); - } else if (_Mchar == _Meta_caret) { // add bol node + } else if (_Mchar == _Meta_caret && (!(_L_flags & _L_anch_rstr) || !_Found)) { // add bol node _Nfa._Add_bol(); _Next(); - _Quant = false; + if ((_L_flags & _L_star_beg) && _Mchar == _Meta_star && !_Found) { + _Nfa._Add_char(_Char); + _Next(); + } else { + _Quant = false; + } } else if (_Mchar == _Meta_dlr) { // add eol node _Nfa._Add_eol(); _Next(); _Quant = false; - } else if (_Mchar == _Meta_star || _Mchar == _Meta_plus || _Mchar == _Meta_query || _Mchar == _Meta_lbr) { + } else if ((_Mchar == _Meta_star && (!(_L_flags & _L_star_beg) || _Found)) || _Mchar == _Meta_plus + || _Mchar == _Meta_query || _Mchar == _Meta_lbr) { _Error(regex_constants::error_badrepeat); } else if (_Mchar == _Meta_rbr && !(_L_flags & _L_paren_bal)) { _Error(regex_constants::error_brace); diff --git a/tests/std/tests/VSO_0000000_regex_use/test.cpp b/tests/std/tests/VSO_0000000_regex_use/test.cpp index 0efb1979e40..bb4d32991f5 100644 --- a/tests/std/tests/VSO_0000000_regex_use/test.cpp +++ b/tests/std/tests/VSO_0000000_regex_use/test.cpp @@ -680,6 +680,331 @@ void test_gh_5160() { neg_regex.should_search_fail(L"xxxYxx\x2009xxxZxxx"); // U+2009 THIN SPACE } +void test_gh_5165_syntax_option(const syntax_option_type basic_or_grep) { + g_regexTester.should_not_match("yx", "y[^x]", basic_or_grep); + g_regexTester.should_match("yz", "y[^x]", basic_or_grep); + g_regexTester.should_match("y^", "y[^x]", basic_or_grep); + + g_regexTester.should_match("yx", "y[x^]", basic_or_grep); + g_regexTester.should_not_match("yz", "y[x^]", basic_or_grep); + g_regexTester.should_match("y^", "y[x^]", basic_or_grep); + + g_regexTester.should_not_match("yx", "y[^x^]", basic_or_grep); + g_regexTester.should_match("yz", "y[^x^]", basic_or_grep); + g_regexTester.should_not_match("y^", "y[^x^]", basic_or_grep); + + { + const test_regex no_anchor(&g_regexTester, "meo[wW]", basic_or_grep); + no_anchor.should_search_match("meow_machine", "meow"); + no_anchor.should_search_match("homeowner", "meow"); + } + { + const test_regex beginning_anchor(&g_regexTester, "^meo[wW]", basic_or_grep); + beginning_anchor.should_search_match("meow_machine", "meow"); + beginning_anchor.should_search_fail("homeowner"); + } + { + const test_regex middle_anchor(&g_regexTester, "me^o[wW]", basic_or_grep); + middle_anchor.should_search_fail("meow_machine"); + middle_anchor.should_search_fail("homeowner"); + middle_anchor.should_search_match("home^owner", "me^ow"); + } + { + const test_regex double_carets(&g_regexTester, "^^meo[wW]", basic_or_grep); + double_carets.should_search_fail("meow_machine"); + double_carets.should_search_fail("homeowner"); + double_carets.should_search_match("^meow_machine", "^meow"); + double_carets.should_search_fail("^^meow_machine"); + double_carets.should_search_fail("ho^meowner"); + double_carets.should_search_fail("ho^^meowner"); + } + + g_regexTester.should_not_match("me^ow", R"(me\(^o[wW]\))", basic_or_grep); + g_regexTester.should_not_match("meow", R"(me\(^o[wW]\))", basic_or_grep); + + { + const test_regex firstgroup_anchor(&g_regexTester, R"(\(^meo[wW]\))", basic_or_grep); + firstgroup_anchor.should_search_match("meow_machine", "meow"); + firstgroup_anchor.should_search_fail("^meow_machine"); + firstgroup_anchor.should_search_fail("homeowner"); + firstgroup_anchor.should_search_fail("ho^meowner"); + } + { + const test_regex prefixedgroup_anchor(&g_regexTester, R"(.*\(^meo[wW]\))", basic_or_grep); + prefixedgroup_anchor.should_search_match("meow_machine", "meow"); + prefixedgroup_anchor.should_search_fail("^meow_machine"); + prefixedgroup_anchor.should_search_fail("homeowner"); + prefixedgroup_anchor.should_search_fail("ho^meowner"); + } + { + const test_regex secondgroup_anchor(&g_regexTester, R"(\(.*\)\(^meo[wW]\))", basic_or_grep); + secondgroup_anchor.should_search_match("meow_machine", "meow"); + secondgroup_anchor.should_search_fail("^meow_machine"); + secondgroup_anchor.should_search_fail("homeowner"); + secondgroup_anchor.should_search_fail("ho^meowner"); + } + { + const test_regex nested_anchor(&g_regexTester, R"(.*\(^\(^meo[wW]\)\))", basic_or_grep); + nested_anchor.should_search_match("meow_machine", "meow"); + nested_anchor.should_search_fail("^meow_machine"); + nested_anchor.should_search_fail("^^meow_machine"); + nested_anchor.should_search_fail("homeowner"); + nested_anchor.should_search_fail("ho^meowner"); + nested_anchor.should_search_fail("ho^^meowner"); + } + { + const test_regex double_carets(&g_regexTester, R"(.*\(^^meo[wW]\))", basic_or_grep); + double_carets.should_search_fail("meow_machine"); + double_carets.should_search_match("^meow_machine", "^meow"); + double_carets.should_search_fail("^^meow_machine"); + double_carets.should_search_fail("homeowner"); + double_carets.should_search_fail("ho^meowner"); + double_carets.should_search_fail("ho^^meowner"); + } + + // Validate correct handling of star at the + // beginning of an expression (with or without optional caret). + g_regexTester.should_match("*", "*", basic_or_grep); + g_regexTester.should_not_match("**", "*", basic_or_grep); + g_regexTester.should_match("****", "**", basic_or_grep); + g_regexTester.should_throw("***", error_badrepeat, basic_or_grep); + + g_regexTester.should_match("*", "^*", basic_or_grep); + g_regexTester.should_not_match("**", "^*", basic_or_grep); + g_regexTester.should_not_match("^*", "^*", basic_or_grep); + g_regexTester.should_match("****", "^**", basic_or_grep); + g_regexTester.should_throw("^***", error_badrepeat, basic_or_grep); + + g_regexTester.should_match("*aa", "*a*", basic_or_grep); + g_regexTester.should_match("*a", "*a*", basic_or_grep); + g_regexTester.should_not_match("aa", "*a*", basic_or_grep); + g_regexTester.should_not_match("*a*", "*a*", basic_or_grep); + + g_regexTester.should_match("*aa", "^*a*", basic_or_grep); + g_regexTester.should_not_match("aa", "^*a*", basic_or_grep); + g_regexTester.should_not_match("*a*", "^*a*", basic_or_grep); + g_regexTester.should_not_match("^*a", "^*a*", basic_or_grep); + g_regexTester.should_not_match("^*aa", "^*a*", basic_or_grep); + g_regexTester.should_not_match("^*a*", "^*a*", basic_or_grep); + + g_regexTester.should_match("*", R"(\(*\))", basic_or_grep); + g_regexTester.should_not_match("**", R"(\(*\))", basic_or_grep); + g_regexTester.should_match("****", R"(\(**\))", basic_or_grep); + g_regexTester.should_throw(R"(\(***\))", error_badrepeat, basic_or_grep); + + g_regexTester.should_match("*", R"(\(^*\))", basic_or_grep); + g_regexTester.should_not_match("**", R"(\(^*\))", basic_or_grep); + g_regexTester.should_not_match("^*", R"(\(^*\))", basic_or_grep); + g_regexTester.should_match("***", R"(\(^**\))", basic_or_grep); + g_regexTester.should_throw(R"(\(^***\))", error_badrepeat, basic_or_grep); + + g_regexTester.should_match("*aa", R"(\(*a*\))", basic_or_grep); + g_regexTester.should_match("*a", R"(\(*a*\))", basic_or_grep); + g_regexTester.should_not_match("aa", R"(\(*a*\))", basic_or_grep); + g_regexTester.should_not_match("*a*", R"(\(*a*\))", basic_or_grep); + + g_regexTester.should_match("*aa", R"(\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("aa", R"(\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("*a*", R"(\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("^*a", R"(\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("^*aa", R"(\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("^*a*", R"(\(^*a*\))", basic_or_grep); + + g_regexTester.should_match("*", R"(.*\(^*\))", basic_or_grep); + g_regexTester.should_not_match("**", R"(.*\(^*\))", basic_or_grep); + g_regexTester.should_not_match("^*", R"(.*\(^*\))", basic_or_grep); + g_regexTester.should_match("***", R"(.*\(^**\))", basic_or_grep); + g_regexTester.should_throw(R"(.*\(^***\))", error_badrepeat, basic_or_grep); + + g_regexTester.should_match("*aa", R"(.*\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("aa", R"(.*\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("*a*", R"(.*\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("^*a", R"(.*\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("^*aa", R"(.*\(^*a*\))", basic_or_grep); + g_regexTester.should_not_match("^*a*", R"(.*\(^*a*\))", basic_or_grep); + + // Validate that there is no special behavior near bars, + // as they are alternation operators in regex modes other than basic or grep. + { + const test_regex middle_bar(&g_regexTester, "^a|a", basic_or_grep); + middle_bar.should_search_match("a|a", "a|a"); + middle_bar.should_search_fail("^a|a"); + middle_bar.should_search_fail("ba|a"); + middle_bar.should_search_fail("a"); + } + { + const test_regex group_middle_bar(&g_regexTester, R"(^\(a|a\))", basic_or_grep); + group_middle_bar.should_search_match("a|a", "a|a"); + group_middle_bar.should_search_fail("^a|a"); + group_middle_bar.should_search_fail("ba|a"); + group_middle_bar.should_search_fail("a"); + } + { + const test_regex middle_bar_with_caret(&g_regexTester, "^a|^b", basic_or_grep); + middle_bar_with_caret.should_search_match("a|^b", "a|^b"); + middle_bar_with_caret.should_search_fail("a|b"); + middle_bar_with_caret.should_search_fail("^a|^b"); + middle_bar_with_caret.should_search_fail("ca|^b"); + middle_bar_with_caret.should_search_fail("a"); + middle_bar_with_caret.should_search_fail("b"); + } + { + const test_regex group_middle_bar_with_caret(&g_regexTester, R"(^\(a|^b\))", basic_or_grep); + group_middle_bar_with_caret.should_search_match("a|^b", "a|^b"); + group_middle_bar_with_caret.should_search_fail("a|b"); + group_middle_bar_with_caret.should_search_fail("^a|^b"); + group_middle_bar_with_caret.should_search_fail("ca|^b"); + group_middle_bar_with_caret.should_search_fail("a"); + group_middle_bar_with_caret.should_search_fail("b"); + } + + g_regexTester.should_match("ab", "a|*b", basic_or_grep); + g_regexTester.should_match("a||b", "a|*b", basic_or_grep); + g_regexTester.should_not_match("a|*b", "a|*b", basic_or_grep); + g_regexTester.should_throw("a|**b", error_badrepeat, basic_or_grep); + + g_regexTester.should_match("ab", "^a|*b", basic_or_grep); + g_regexTester.should_match("a||b", "^a|*b", basic_or_grep); + g_regexTester.should_not_match("a|*b", "^a|*b", basic_or_grep); + g_regexTester.should_throw("^a|**b", error_badrepeat, basic_or_grep); + + g_regexTester.should_match("a|b", "^a|^*b", basic_or_grep); + g_regexTester.should_match("a|^^b", "^a|^*b", basic_or_grep); + g_regexTester.should_not_match("a|*b", "^a|^*b", basic_or_grep); + g_regexTester.should_not_match("a|^*b", "^a|^*b", basic_or_grep); + g_regexTester.should_throw("^a|^**b", error_badrepeat, basic_or_grep); +} + +void test_gh_5165_basic() { + // test cases specific for basic regular expressions + { + const test_regex middle_nl(&g_regexTester, "^a\na", basic); + middle_nl.should_search_match("a\na", "a\na"); + middle_nl.should_search_fail("^a\na"); + middle_nl.should_search_fail("ba\na"); + middle_nl.should_search_fail("a"); + } + { + const test_regex group_middle_nl(&g_regexTester, "^\\(a\na\\)", basic); + group_middle_nl.should_search_match("a\na", "a\na"); + group_middle_nl.should_search_fail("^a\na"); + group_middle_nl.should_search_fail("ba\na"); + group_middle_nl.should_search_fail("a"); + } + { + const test_regex middle_nl_with_caret(&g_regexTester, "^a\n^b", basic); + middle_nl_with_caret.should_search_match("a\n^b", "a\n^b"); + middle_nl_with_caret.should_search_fail("a\nb"); + middle_nl_with_caret.should_search_fail("^a\n^b"); + middle_nl_with_caret.should_search_fail("ca\n^b"); + middle_nl_with_caret.should_search_fail("a"); + middle_nl_with_caret.should_search_fail("b"); + } + { + const test_regex group_middle_nl_with_caret(&g_regexTester, "^\\(a\n^b\\)", basic); + group_middle_nl_with_caret.should_search_match("a\n^b", "a\n^b"); + group_middle_nl_with_caret.should_search_fail("a\nb"); + group_middle_nl_with_caret.should_search_fail("^a\n^b"); + group_middle_nl_with_caret.should_search_fail("ca\n^b"); + group_middle_nl_with_caret.should_search_fail("a"); + group_middle_nl_with_caret.should_search_fail("b"); + } + + g_regexTester.should_match("ab", "a\n*b", basic); + g_regexTester.should_match("a\n\nb", "a\n*b", basic); + g_regexTester.should_not_match("a\n*b", "a\n*b", basic); + g_regexTester.should_match("a\n\nb", "^a\n*b", basic); + g_regexTester.should_throw("^a\n**b", error_badrepeat, basic); + + g_regexTester.should_match("a\nb", "^a\n^*b", basic); + g_regexTester.should_match("a\n^^b", "^a\n^*b", basic); + g_regexTester.should_not_match("a\n*b", "^a\n^*b", basic); + g_regexTester.should_not_match("a\n^*b", "^a\n^*b", basic); + g_regexTester.should_throw("^a\n^**b", error_badrepeat, basic); +} + +void test_gh_5165_grep() { + // test cases specific for grep mode + { + const test_regex middle_nl(&g_regexTester, "^a\na", grep); + middle_nl.should_search_match("a\na", "a"); + middle_nl.should_search_match("^a\na", "a"); + middle_nl.should_search_match("ba\na", "a"); + middle_nl.should_search_match("a", "a"); + middle_nl.should_search_fail("b"); + } + { + // This regular expression is not accepted by POSIX grep, but currently the regex parser does not reject it. + // If the parser is changed to reject it, adjust this test case. + const test_regex group_middle_nl(&g_regexTester, "^\\(a\na\\)", grep); + group_middle_nl.should_search_match("a\na", "a\na"); + group_middle_nl.should_search_fail("^a\na"); + group_middle_nl.should_search_fail("ba\na"); + group_middle_nl.should_search_fail("a"); + } + { + const test_regex middle_nl_with_caret(&g_regexTester, "^a\n^b", grep); + middle_nl_with_caret.should_search_match("a\n^b", "a"); + middle_nl_with_caret.should_search_match("a\nb", "a"); + middle_nl_with_caret.should_search_match("ab", "a"); + middle_nl_with_caret.should_search_match("a", "a"); + middle_nl_with_caret.should_search_match("b", "b"); + middle_nl_with_caret.should_search_match("ba", "b"); + middle_nl_with_caret.should_search_fail("^a"); + middle_nl_with_caret.should_search_fail("ca"); + middle_nl_with_caret.should_search_fail("^b"); + middle_nl_with_caret.should_search_fail("ca"); + middle_nl_with_caret.should_search_fail("cb"); + } + { + // This regular expression is not accepted by POSIX grep, but currently the regex parser does not reject it. + // If the parser is changed to reject it, adjust this test case. + const test_regex group_middle_nl_with_caret(&g_regexTester, "^\\(a\n^b\\)", grep); + group_middle_nl_with_caret.should_search_match("a\n^b", "a\n^b"); + group_middle_nl_with_caret.should_search_fail("a\nb"); + group_middle_nl_with_caret.should_search_fail("^a\n^b"); + group_middle_nl_with_caret.should_search_fail("ca\n^b"); + group_middle_nl_with_caret.should_search_fail("a"); + group_middle_nl_with_caret.should_search_fail("b"); + } + + g_regexTester.should_not_match("ab", "a\n*b", grep); + g_regexTester.should_not_match("a\n\nb", "a\n*b", grep); + g_regexTester.should_not_match("a\n*b", "a\n*b", grep); + g_regexTester.should_match("a", "a\n*b", grep); + g_regexTester.should_match("*b", "a\n*b", grep); + g_regexTester.should_match("a", "a\n**b", grep); + g_regexTester.should_match("***b", "a\n**b", grep); + + g_regexTester.should_not_match("ab", "^a\n*b", grep); + g_regexTester.should_not_match("a\n\nb", "^a\n*b", grep); + g_regexTester.should_not_match("a\n*b", "^a\n*b", grep); + g_regexTester.should_match("a", "^a\n*b", grep); + g_regexTester.should_match("*b", "^a\n*b", grep); + g_regexTester.should_match("a", "^a\n**b", grep); + g_regexTester.should_match("****b", "^a\n**b", grep); + + g_regexTester.should_not_match("a\nb", "^a\n^*b", grep); + g_regexTester.should_not_match("a\n^^b", "^a\n^*b", grep); + g_regexTester.should_not_match("a\n*b", "^a\n^*b", grep); + g_regexTester.should_not_match("a\n^*b", "^a\n^*b", grep); + g_regexTester.should_not_match("^*b", "^a\n^*b", grep); + g_regexTester.should_match("a", "^a\n^*b", grep); + g_regexTester.should_match("*b", "^a\n^*b", grep); + g_regexTester.should_not_match("**b", "^a\n^*b", grep); + g_regexTester.should_match("a", "^a\n^**b", grep); + g_regexTester.should_match("****b", "^a\n^**b", grep); +} + +void test_gh_5165() { + // GH-5165: Revise caret parsing in basic and grep mode + test_gh_5165_syntax_option(basic); + test_gh_5165_syntax_option(grep); + + test_gh_5165_basic(); + test_gh_5165_grep(); +} + void test_gh_5167() { // GH-5167: Limit backreference parsing to single digit for basic regular expressions g_regexTester.should_match("abab0", R"(\(ab*\)\10)", basic); @@ -779,6 +1104,7 @@ int main() { test_gh_4995(); test_gh_5058(); test_gh_5160(); + test_gh_5165(); test_gh_5167(); test_gh_5192(); test_gh_5214(); From a7a5b878566d621223048db5c30304d892764702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Tue, 25 Mar 2025 00:47:40 +0100 Subject: [PATCH 08/14] ``: Fix depth-first and leftmost-longest matching rules (#5218) Co-authored-by: Stephan T. Lavavej --- stl/inc/regex | 39 ++++++-- tests/std/include/test_regex_support.hpp | 90 +++++++++++++++++++ .../std/tests/VSO_0000000_regex_use/test.cpp | 79 ++++++++++++++++ tests/tr1/tests/regex2/test.cpp | 6 +- 4 files changed, 204 insertions(+), 10 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index 0d1d5b84f5d..1f751b0adba 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -1638,11 +1638,12 @@ public: if (_Matches) { // copy results to _Matches _Matches->_Resize(_Get_ncap()); + const auto& _Result = _Longest ? _Res : _Tgt_state; for (unsigned int _Idx = 0; _Idx < _Get_ncap(); ++_Idx) { // copy submatch _Idx - if (_Res._Grp_valid[_Idx]) { // copy successful match + if (_Result._Grp_valid[_Idx]) { // copy successful match _Matches->_At(_Idx).matched = true; - _Matches->_At(_Idx).first = _Res._Grps[_Idx]._Begin; - _Matches->_At(_Idx).second = _Res._Grps[_Idx]._End; + _Matches->_At(_Idx).first = _Result._Grps[_Idx]._Begin; + _Matches->_At(_Idx).second = _Result._Grps[_Idx]._End; } else { // copy failed match _Matches->_At(_Idx).matched = false; _Matches->_At(_Idx).first = _End; @@ -3277,6 +3278,20 @@ bool _Matcher<_BidIt, _Elem, _RxTraits, _It>::_Do_rep(_Node_rep* _Node, bool _Gr _Psav->_Loop_iter = _STD addressof(_Cur_iter); _Matched0 = _Match_pat(_Node->_Next); } + } else if (_Longest) { // longest, try any number of repetitions + + // match with no further repetition + _Matched0 = _Match_pat(_Node->_End_rep->_Next); + // match with at least one more repetition if last repetition made progress + if (_Progress) { + _Tgt_state = _St; + _Psav->_Loop_idx = _Init_idx + 1; + _Psav->_Loop_iter = _STD addressof(_Cur_iter); + + if (_Match_pat(_Node->_Next)) { // always call _Match_pat, even when _Matched0 is already true + _Matched0 = true; + } + } } else if (!_Greedy) { // not greedy, favor minimum number of reps _Matched0 = _Match_pat(_Node->_End_rep->_Next); if (!_Matched0 && _Progress) { // tail failed, try another rep @@ -3437,16 +3452,24 @@ bool _Matcher<_BidIt, _Elem, _RxTraits, _It>::_Do_class(_Node_base* _Nx) { // ap } template -bool _Matcher<_BidIt, _Elem, _RxTraits, _It>::_Better_match() { // check for better match under UNIX rules +bool _Matcher<_BidIt, _Elem, _RxTraits, _It>::_Better_match() { // check for better match under leftmost-longest rule for (unsigned int _Ix = 0; _Ix < _Get_ncap(); ++_Ix) { // check each capture group - if (_Res._Grp_valid[_Ix] && _Tgt_state._Grp_valid[_Ix]) { + // any match (even an empty one) is better than no match at all + if (_Res._Grp_valid[_Ix] != _Tgt_state._Grp_valid[_Ix]) { + return _Tgt_state._Grp_valid[_Ix]; + } + + if (_Res._Grp_valid[_Ix]) { // now known to be equal to _Tgt_state._Grp_valid[_Ix], no need to test both + // if both groups are matched, prefer the leftmost one if (_Res._Grps[_Ix]._Begin != _Tgt_state._Grps[_Ix]._Begin) { return _STD distance(_Begin, _Res._Grps[_Ix]._Begin) - < _STD distance(_Begin, _Tgt_state._Grps[_Ix]._Begin); + > _STD distance(_Begin, _Tgt_state._Grps[_Ix]._Begin); } + // if both groups start at the same position, prefer the longer one if (_Res._Grps[_Ix]._End != _Tgt_state._Grps[_Ix]._End) { - return _STD distance(_Begin, _Res._Grps[_Ix]._End) < _STD distance(_Begin, _Tgt_state._Grps[_Ix]._End); + return _STD distance(_Res._Grps[_Ix]._Begin, _Res._Grps[_Ix]._End) + < _STD distance(_Tgt_state._Grps[_Ix]._Begin, _Tgt_state._Grps[_Ix]._End); } } } @@ -3665,7 +3688,7 @@ bool _Matcher<_BidIt, _Elem, _RxTraits, _It>::_Match_pat(_Node_base* _Nx) { // c && _Begin == _Tgt_state._Cur) || (_Full && _Tgt_state._Cur != _End)) { _Failed = true; - } else if (!_Matched || _Better_match()) { // record successful match + } else if (_Longest && (!_Matched || _Better_match())) { // record successful match _Res = _Tgt_state; _Matched = true; } diff --git a/tests/std/include/test_regex_support.hpp b/tests/std/include/test_regex_support.hpp index d45b77dec8a..523b3535b30 100644 --- a/tests/std/include/test_regex_support.hpp +++ b/tests/std/include/test_regex_support.hpp @@ -2,9 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #pragma once +#include #include +#include #include #include +#include class regex_fixture { int regex_test_result = 0; @@ -241,6 +244,93 @@ class test_regex { fixture->fail_regex(); } } + + void should_search_match_capture_groups(const std::string& subject, const std::string& expected, + const std::regex_constants::match_flag_type match_flags, + std::initializer_list> capture_groups) const { + std::smatch mr; + try { + const bool search_result = std::regex_search(subject, mr, r, match_flags); + if (!search_result || mr[0] != expected) { + printf(R"(Expected regex_search("%s", regex("%s", 0x%X), 0x%X) to find "%s", )", subject.c_str(), + pattern.c_str(), static_cast(syntax), static_cast(match_flags), + expected.c_str()); + if (search_result) { + printf(R"(but it matched "%s")" + "\n", + mr.str().c_str()); + } else { + puts("but it failed to match"); + } + + fixture->fail_regex(); + } else if (capture_groups.size() + 1 != mr.size()) { + printf(R"(Expected regex_search("%s", regex("%s", 0x%X), 0x%X) to match %zu capture groups in "%s", )", + subject.c_str(), pattern.c_str(), static_cast(syntax), + static_cast(match_flags), capture_groups.size() + 1, expected.c_str()); + printf(R"(but it matched %zu groups)" + "\n", + mr.size()); + fixture->fail_regex(); + } else { + bool submatches_success = true; + for (std::size_t i = 1U; i < mr.size(); ++i) { + const auto& expected_capture = capture_groups.begin()[i - 1]; + const auto& actual_capture = mr[i]; + if (expected_capture.first == -1) { + if (actual_capture.matched) { + submatches_success = false; + break; + } + } else if (!actual_capture.matched || actual_capture.first != (mr[0].first + expected_capture.first) + || actual_capture.second != (mr[0].first + expected_capture.second)) { + submatches_success = false; + break; + } + } + if (!submatches_success) { + printf(R"(Expected regex_search("%s", regex("%s", 0x%X), 0x%X) to find capture groups {)", + subject.c_str(), pattern.c_str(), static_cast(syntax), + static_cast(match_flags)); + + bool initial = true; + for (const auto& expected_capture : capture_groups) { + std::string capture = "(unmatched)"; + if (expected_capture.first != -1) { + capture.assign(mr[0].first + expected_capture.first, mr[0].first + expected_capture.second); + } + printf(R"(%s"%s" [%td %td])", initial ? "" : ", ", capture.c_str(), expected_capture.first, + expected_capture.second); + initial = false; + } + printf(R"(} in "%s", but found {)", expected.c_str()); + + initial = true; + for (std::size_t i = 1U; i < mr.size(); ++i) { + const auto& actual_capture = mr[i]; + std::string capture = "(unmatched)"; + std::ptrdiff_t first = -1; + std::ptrdiff_t last = -1; + if (actual_capture.matched) { + capture = actual_capture.str(); + first = actual_capture.first - mr[0].first; + last = actual_capture.second - mr[0].first; + } + printf(R"(%s"%s" [%td %td])", initial ? "" : ", ", capture.c_str(), first, last); + initial = false; + } + printf("}\n"); + fixture->fail_regex(); + } + } + } catch (const std::regex_error& e) { + printf(R"(Failed to regex_search("%s", regex("%s", 0x%X), 0x%X): regex_error: "%s")" + "\n", + subject.c_str(), pattern.c_str(), static_cast(syntax), + static_cast(match_flags), e.what()); + fixture->fail_regex(); + } + } }; class test_wregex { diff --git a/tests/std/tests/VSO_0000000_regex_use/test.cpp b/tests/std/tests/VSO_0000000_regex_use/test.cpp index bb4d32991f5..a98b6ff9122 100644 --- a/tests/std/tests/VSO_0000000_regex_use/test.cpp +++ b/tests/std/tests/VSO_0000000_regex_use/test.cpp @@ -558,6 +558,84 @@ void test_construction_from_nullptr_and_zero() { } } +void test_gh_731() { + // GH-731 : Incorrect behavior for capture groups + // GH-996: regex_search behaves incorrectly when the regex contains R"(\[)" + + // Several bugs were fixed in ECMAScript (depth-first) and POSIX (leftmost-longest) matching rules. + { + const test_regex ecma_regex(&g_regexTester, R"((A+)\s*(B+)?\s*B*)", ECMAScript); + ecma_regex.should_search_match_capture_groups("AAA BBB", "AAA BBB", match_default, {{0, 3}, {4, 7}}); + } + for (syntax_option_type option : {extended, egrep, awk}) { + const test_regex posix_regex(&g_regexTester, R"((A+)[[:space:]]*(B+)?[[:space:]]*B*)", option); + posix_regex.should_search_match_capture_groups("AAA BBB", "AAA BBB", match_default, {{0, 3}, {4, 7}}); + } + + { + const test_regex ecma_regex(&g_regexTester, ".*(cat|concatenate)", ECMAScript); + ecma_regex.should_search_match_capture_groups("WXconcatenateYZ", "WXconcat", match_default, {{5, 8}}); + } + for (syntax_option_type option : {extended, egrep, awk}) { + const test_regex posix_regex(&g_regexTester, ".*(cat|concatenate)", option); + posix_regex.should_search_match_capture_groups("WXconcatenateYZ", "WXconcatenate", match_default, {{2, 13}}); + } + + { + const test_regex ecma_regex(&g_regexTester, "(aa|aabaac|ba|b|c)*", ECMAScript); + ecma_regex.should_search_match_capture_groups("aabaac", "aaba", match_default, {{2, 4}}); + } + for (syntax_option_type option : {extended, egrep, awk}) { + const test_regex posix_regex(&g_regexTester, "(aa|aabaac|ba|b|c)*", option); + posix_regex.should_search_match_capture_groups("aabaac", "aabaac", match_default, {{0, 6}}); + } + + { + const test_regex ecma_regex(&g_regexTester, ".*(a|bacc|baccc)", ECMAScript); + ecma_regex.should_search_match_capture_groups("ddbacccd", "ddba", match_default, {{3, 4}}); + } + { + const test_regex ecma_regex(&g_regexTester, ".*?(a|bacc|baccc)", ECMAScript); + ecma_regex.should_search_match_capture_groups("ddbacccd", "ddbacc", match_default, {{2, 6}}); + } + for (syntax_option_type option : {extended, egrep, awk}) { + const test_regex posix_regex(&g_regexTester, ".*(a|bacc|baccc)", option); + posix_regex.should_search_match_capture_groups("ddbacccd", "ddbaccc", match_default, {{2, 7}}); + } + + { + const test_regex ecma_regex(&g_regexTester, R"(^[[:blank:]]*#([^\n]*\\[[:space:]]+)*[^\n]*)", ECMAScript); + ecma_regex.should_search_match_capture_groups("#define some_symbol(x) \\ \r\n cat();\\\r\n printf(#x);", + "#define some_symbol(x) \\ \r\n cat();\\\r\n printf(#x);", match_default, {{30, 42}}); + } + { + const test_regex awk_regex(&g_regexTester, R"(^[[:blank:]]*#([^\n]*\\[[:space:]]+)*[^\n]*)", awk); + awk_regex.should_search_match_capture_groups("#define some_symbol(x) \\ \r\n cat();\\\r\n printf(#x);", + "#define some_symbol(x) \\ \r\n cat();\\\r\n printf(#x);", match_default, {{28, 42}}); + } + { + const test_regex extended_regex(&g_regexTester, "^[[:blank:]]*#([^\n]*\\\\[[:space:]]+)*[^\n]*", extended); + extended_regex.should_search_match_capture_groups("#define some_symbol(x) \\ \r\n cat();\\\r\n printf(#x);", + "#define some_symbol(x) \\ \r\n cat();\\\r\n printf(#x);", match_default, {{28, 42}}); + } + + { + const test_regex ecma_regex(&g_regexTester, "(ab*)*(ce|bbceef)", ECMAScript); + ecma_regex.should_search_match_capture_groups("aababbbceef", "aababbbce", match_default, {{3, 7}, {7, 9}}); + } + for (syntax_option_type option : {extended, egrep, awk}) { + const test_regex posix_regex(&g_regexTester, "(ab*)*(ce|bbceef)", option); + posix_regex.should_search_match_capture_groups("aababbbceef", "aababbbceef", match_default, {{3, 5}, {5, 11}}); + } + + { + // GH-996 test case + const test_regex ecma_regex(&g_regexTester, R"( *((<<)|(\[)|(.+)))", ECMAScript); + ecma_regex.should_search_match_capture_groups( + " [<>]>>", " [", match_default, {{1, 2}, {-1, -1}, {1, 2}, {-1, -1}}); + } +} + void test_gh_993() { // GH-993 regex::icase is not handled correctly for some input. { @@ -1100,6 +1178,7 @@ int main() { test_VSO_225160_match_eol_flag(); test_VSO_226914_word_boundaries(); test_construction_from_nullptr_and_zero(); + test_gh_731(); test_gh_993(); test_gh_4995(); test_gh_5058(); diff --git a/tests/tr1/tests/regex2/test.cpp b/tests/tr1/tests/regex2/test.cpp index 5d676d4fd8e..b720710ff9f 100644 --- a/tests/tr1/tests/regex2/test.cpp +++ b/tests/tr1/tests/regex2/test.cpp @@ -659,7 +659,7 @@ static const regex_test tests[] = { {__LINE__, T("a[a-z]\\{2,4\\}"), T("abcdefghi"), "1 0 5", BASIC | GREP}, {__LINE__, T("a[a-z]{2,4}?"), T("abcdefghi"), "1 0 3", ECMA}, {__LINE__, T("(aa|aabaac|ba|b|c)*"), T("aabaac"), "2 0 4 2 4", ECMA}, - {__LINE__, T("(aa|aabaac|ba|b|c)*"), T("aabaac"), "2 0 6 5 6", EEA}, + {__LINE__, T("(aa|aabaac|ba|b|c)*"), T("aabaac"), "2 0 6 0 6", EEA}, {__LINE__, T("(z)((a+)?(b+)?(c))*"), T("zaacbbbcac"), "6 0 10 0 1 8 10 8 9 -1 -1 9 10", ECMA}, {__LINE__, T("(a*)b\\1+"), T("baaaac"), "2 0 1 0 0", ECMA}, {__LINE__, T("(?=(a+))"), T("baaabac"), "2 1 1 1 4", ECMA}, @@ -774,7 +774,9 @@ static const regex_test tests[] = { {__LINE__, T("^[[:blank:]]*#([^\\n]*\\\\[[:space:]]+)*[^\\n]*"), T("#define some_symbol(x) #x"), "2 0 25 -1 -1", ECMA | AWK}, {__LINE__, T("^[[:blank:]]*#([^\\n]*\\\\[[:space:]]+)*[^\\n]*"), - T("#define some_symbol(x) \\ \r\n cat();\\\r\n printf(#x);"), "2 0 53 30 42", ECMA | AWK}, + T("#define some_symbol(x) \\ \r\n cat();\\\r\n printf(#x);"), "2 0 53 30 42", ECMA}, + {__LINE__, T("^[[:blank:]]*#([^\\n]*\\\\[[:space:]]+)*[^\\n]*"), + T("#define some_symbol(x) \\ \r\n cat();\\\r\n printf(#x);"), "2 0 53 28 42", AWK}, }; static STD string check_matches( From 92cbb9b38054b34d720c655f3e1ffd3e6bbb0054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Tue, 25 Mar 2025 00:52:08 +0100 Subject: [PATCH 09/14] ``: Implement collating ranges (#5238) Co-authored-by: Stephan T. Lavavej --- stl/inc/regex | 85 +++-- tests/std/test.lst | 1 + .../GH_005204_regex_collating_ranges/env.lst | 4 + .../GH_005204_regex_collating_ranges/test.cpp | 345 ++++++++++++++++++ 4 files changed, 405 insertions(+), 30 deletions(-) create mode 100644 tests/std/tests/GH_005204_regex_collating_ranges/env.lst create mode 100644 tests/std/tests/GH_005204_regex_collating_ranges/test.cpp diff --git a/stl/inc/regex b/stl/inc/regex index 1f751b0adba..1393336ac78 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -1721,7 +1721,7 @@ public: private: // lexing - void _Error(regex_constants::error_type); + [[noreturn]] void _Error(regex_constants::error_type); bool _Is_esc() const; void _Trans(); @@ -2912,7 +2912,8 @@ void _Builder<_FwdIt, _Elem, _RxTraits>::_Add_range2(const _Elem _Arg0, const _E _Node->_Small->_Mark(_Ex0); } - if (_Ex1 >= _Ex0) { + + if ((_Flags & regex_constants::collate) || _Ex1 >= _Ex0) { if (_Ex1 - _Ex0 < _Get_tmax()) { for (; _Ex0 <= _Ex1; ++_Ex0) { _Add_char_to_array(static_cast<_Elem>(_Ex0)); @@ -3364,6 +3365,20 @@ bool _Lookup_range(unsigned int _Ch, const _Buf<_Elem>* _Bufptr) { // check whet return false; } +template +bool _Lookup_collating_range(const _Elem _Ch, const _Buf<_Elem>* const _Bufptr, const _RxTraits& _Traits) { + const typename _RxTraits::string_type _Str = _Traits.transform(_STD addressof(_Ch), _STD addressof(_Ch) + 1); + for (unsigned int _Ix = 0; _Ix < _Bufptr->_Size(); _Ix += 2) { // check current position + const _Elem _Left = _Bufptr->_At(_Ix); + const _Elem _Right = _Bufptr->_At(_Ix + 1); + if (_Traits.transform(_STD addressof(_Left), _STD addressof(_Left) + 1) <= _Str + && _Str <= _Traits.transform(_STD addressof(_Right), _STD addressof(_Right) + 1)) { + return true; + } + } + return false; +} + template bool _Lookup_equiv(typename _RxTraits::_Uelem _Ch, const _Sequence<_Elem>* _Eq, const _RxTraits& _Traits) { // check whether _Ch is in _Eq @@ -3407,35 +3422,36 @@ _BidIt _Lookup_coll(_BidIt _First, _BidIt _Last, const _Sequence<_Elem>* _Eq) { template bool _Matcher<_BidIt, _Elem, _RxTraits, _It>::_Do_class(_Node_base* _Nx) { // apply bracket expression bool _Found; - auto _Ch = static_cast(*_Tgt_state._Cur); + _Elem _Ch = *_Tgt_state._Cur; if (_Sflags & regex_constants::icase) { - _Ch = static_cast(_Traits.translate_nocase(static_cast<_Elem>(_Ch))); + _Ch = _Traits.translate_nocase(_Ch); + } else if (_Sflags & regex_constants::collate) { + _Ch = _Traits.translate(_Ch); } + const auto _UCh = static_cast(_Ch); _It _Res0 = _Tgt_state._Cur; ++_Res0; _It _Resx; _Node_class<_Elem, _RxTraits>* _Node = static_cast<_Node_class<_Elem, _RxTraits>*>(_Nx); if (_Node->_Coll - && (_Resx = _Lookup_coll(_Tgt_state._Cur, _End, _Node->_Coll)) + && (_Resx = _STD _Lookup_coll(_Tgt_state._Cur, _End, _Node->_Coll)) != _Tgt_state._Cur) { // check for collation element _Res0 = _Resx; _Found = true; } else if (_Node->_Ranges - && (_Lookup_range(static_cast( - _Sflags & regex_constants::collate ? _Traits.translate(static_cast<_Elem>(_Ch)) - : static_cast<_Elem>(_Ch)), - _Node->_Ranges))) { + && (_Sflags & regex_constants::collate ? _STD _Lookup_collating_range(_Ch, _Node->_Ranges, _Traits) + : _STD _Lookup_range(_UCh, _Node->_Ranges))) { _Found = true; - } else if (_Ch < _Bmp_max) { - _Found = _Node->_Small && _Node->_Small->_Find(_Ch); + } else if (_UCh < _Bmp_max) { + _Found = _Node->_Small && _Node->_Small->_Find(_UCh); } else if (_Node->_Large && _STD find(_Node->_Large->_Str(), _Node->_Large->_Str() + _Node->_Large->_Size(), _Ch) != _Node->_Large->_Str() + _Node->_Large->_Size()) { _Found = true; - } else if (_Node->_Classes != 0 && _Traits.isctype(static_cast<_Elem>(_Ch), _Node->_Classes)) { + } else if (_Node->_Classes != typename _RxTraits::char_class_type{} && _Traits.isctype(_Ch, _Node->_Classes)) { _Found = true; - } else if (_Node->_Equiv && _Lookup_equiv(_Ch, _Node->_Equiv, _Traits)) { + } else if (_Node->_Equiv && _STD _Lookup_equiv(_UCh, _Node->_Equiv, _Traits)) { _Found = true; } else { _Found = false; @@ -3762,34 +3778,36 @@ _BidIt _Matcher<_BidIt, _Elem, _RxTraits, _It>::_Skip(_BidIt _First_arg, _BidIt case _N_class: { // check for string match for (; _First_arg != _Last; ++_First_arg) { // look for starting match - using _Uelem = typename _RxTraits::_Uelem; bool _Found; - auto _Ch = static_cast<_Uelem>(*_First_arg); + _Elem _Ch = *_First_arg; + if (_Sflags & regex_constants::icase) { + _Ch = _Traits.translate_nocase(_Ch); + } else if (_Sflags & regex_constants::collate) { + _Ch = _Traits.translate(_Ch); + } + const auto _UCh = static_cast(_Ch); + _Node_class<_Elem, _RxTraits>* _Node = static_cast<_Node_class<_Elem, _RxTraits>*>(_Nx); _It _Next = _First_arg; ++_Next; - if (_Sflags & regex_constants::icase) { - _Ch = static_cast<_Uelem>(_Traits.translate_nocase(static_cast<_Elem>(_Ch))); - } - - if (_Node->_Coll && _Lookup_coll(_First_arg, _Next, _Node->_Coll) != _First_arg) { + if (_Node->_Coll && _STD _Lookup_coll(_First_arg, _Next, _Node->_Coll) != _First_arg) { _Found = true; } else if (_Node->_Ranges - && (_Lookup_range(static_cast<_Uelem>(_Sflags & regex_constants::collate - ? _Traits.translate(static_cast<_Elem>(_Ch)) - : static_cast<_Elem>(_Ch)), - _Node->_Ranges))) { + && (_Sflags & regex_constants::collate + ? _STD _Lookup_collating_range(_Ch, _Node->_Ranges, _Traits) + : _STD _Lookup_range(_UCh, _Node->_Ranges))) { _Found = true; - } else if (_Ch < _Bmp_max) { - _Found = _Node->_Small && _Node->_Small->_Find(_Ch); + } else if (_UCh < _Bmp_max) { + _Found = _Node->_Small && _Node->_Small->_Find(_UCh); } else if (_Node->_Large && _STD find(_Node->_Large->_Str(), _Node->_Large->_Str() + _Node->_Large->_Size(), _Ch) != _Node->_Large->_Str() + _Node->_Large->_Size()) { _Found = true; - } else if (_Node->_Classes && _Traits.isctype(static_cast<_Elem>(_Ch), _Node->_Classes)) { + } else if (_Node->_Classes != typename _RxTraits::char_class_type{} + && _Traits.isctype(_Ch, _Node->_Classes)) { _Found = true; - } else if (_Node->_Equiv && _Lookup_equiv(_Ch, _Node->_Equiv, _Traits)) { + } else if (_Node->_Equiv && _STD _Lookup_equiv(_UCh, _Node->_Equiv, _Traits)) { _Found = true; } else { _Found = false; @@ -3858,7 +3876,7 @@ _BidIt _Matcher<_BidIt, _Elem, _RxTraits, _It>::_Skip(_BidIt _First_arg, _BidIt } template -void _Parser<_FwdIt, _Elem, _RxTraits>::_Error(regex_constants::error_type _Code) { // handle error +[[noreturn]] void _Parser<_FwdIt, _Elem, _RxTraits>::_Error(regex_constants::error_type _Code) { // handle error _Xregex_error(_Code); } @@ -4171,7 +4189,14 @@ void _Parser<_FwdIt, _Elem, _RxTraits>::_ClassRanges() { // check for valid clas _Chr2 = _Traits.translate(_Chr2); } - if (static_cast(_Chr2) < static_cast(_Chr1)) { + if (_Flags & regex_constants::collate) { + const _Elem* const _Chr1_ptr = _STD addressof(_Chr1); + const _Elem* const _Chr2_ptr = _STD addressof(_Chr2); + if (_Traits.transform(_Chr2_ptr, _Chr2_ptr + 1) < _Traits.transform(_Chr1_ptr, _Chr1_ptr + 1)) { + _Error(regex_constants::error_range); + } + } else if (static_cast(_Chr2) + < static_cast(_Chr1)) { _Error(regex_constants::error_range); } diff --git a/tests/std/test.lst b/tests/std/test.lst index 484a11f3a14..f4b02966ba2 100644 --- a/tests/std/test.lst +++ b/tests/std/test.lst @@ -254,6 +254,7 @@ tests\GH_004845_logical_operator_traits_with_non_bool_constant tests\GH_004929_internal_tag_constructors tests\GH_004930_char_traits_user_specialization tests\GH_005090_stl_hardening +tests\GH_005204_regex_collating_ranges tests\GH_005315_destructor_tombstones tests\LWG2381_num_get_floating_point tests\LWG2597_complex_branch_cut diff --git a/tests/std/tests/GH_005204_regex_collating_ranges/env.lst b/tests/std/tests/GH_005204_regex_collating_ranges/env.lst new file mode 100644 index 00000000000..19f025bd0e6 --- /dev/null +++ b/tests/std/tests/GH_005204_regex_collating_ranges/env.lst @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +RUNALL_INCLUDE ..\usual_matrix.lst diff --git a/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp new file mode 100644 index 00000000000..afa5b36d162 --- /dev/null +++ b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp @@ -0,0 +1,345 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include + +#include + +// skip collation tests when linking to the DLL in case of +// * undefined _NATIVE_WCHAR_T_DEFINED due to GH-5236 +// * _ITERATOR_DEBUG_LEVEL mismatch between code and linked DLL +#ifdef _DEBUG +#define DEFAULT_IDL_SETTING 2 +#else +#define DEFAULT_IDL_SETTING 0 +#endif + +#ifdef _DLL +#ifndef _NATIVE_WCHAR_T_DEFINED // TRANSITION, GH-212 or GH-5236 +#define SKIP_COLLATE_TESTS +#elif _ITERATOR_DEBUG_LEVEL != DEFAULT_IDL_SETTING +#define SKIP_COLLATE_TESTS +#endif // !defined(_NATIVE_WCHAR_T_DEFINED) || _ITERATOR_DEBUG_LEVEL != DEFAULT_IDL_SETTING +#endif // defined(_DLL) + + +using namespace std; +using namespace std::regex_constants; + +class test_wregex_locale { +private: + regex_fixture* const fixture; + const wstring pattern; + const string locname; + const syntax_option_type syntax; + wregex r; + +public: + test_wregex_locale(regex_fixture* const fixture_, const wstring& pattern_, const string& locname_, + const syntax_option_type syntax_ = ECMAScript) + : fixture(fixture_), pattern(pattern_), locname(locname_), syntax(syntax_), r() { + r.imbue(locale{locname}); + r.assign(pattern, syntax); + } + + test_wregex_locale(const test_wregex_locale&) = delete; + test_wregex_locale& operator=(const test_wregex_locale&) = delete; + + void should_search_match( + const wstring& subject, const wstring& expected, const match_flag_type match_flags = match_default) const { + wsmatch mr; + try { + const bool search_result = regex_search(subject, mr, r, match_flags); + if (!search_result || mr[0] != expected) { + wprintf(LR"(Expected regex_search("%s", regex("%s", 0x%X), 0x%X) to find "%s" for locale "%hs", )", + subject.c_str(), pattern.c_str(), static_cast(syntax), + static_cast(match_flags), expected.c_str(), locname.c_str()); + if (search_result) { + wprintf(LR"(but it matched "%s")" + "\n", + mr.str().c_str()); + } else { + puts("but it failed to match"); + } + + fixture->fail_regex(); + } + } catch (const regex_error& e) { + wprintf(LR"(Failed to regex_search("%s", regex("%s", 0x%X), 0x%X))", subject.c_str(), pattern.c_str(), + static_cast(syntax), static_cast(match_flags)); + printf(R"( for locale "%s": regex_error: "%s")" + "\n", + locname.c_str(), e.what()); + fixture->fail_regex(); + } + } + + void should_search_fail(const wstring& subject, const match_flag_type match_flags = match_default) const { + wsmatch mr; + try { + if (regex_search(subject, mr, r, match_flags)) { + wprintf(LR"(Expected regex_search("%s", regex("%s", 0x%X), 0x%X) to not match )" + LR"(for locale "%hs", but it found "%s")" + "\n", + subject.c_str(), pattern.c_str(), static_cast(syntax), + static_cast(match_flags), locname.c_str(), mr.str().c_str()); + fixture->fail_regex(); + } + } catch (const regex_error& e) { + wprintf(LR"(Failed to regex_search("%s", regex("%s", 0x%X), 0x%X))", subject.c_str(), pattern.c_str(), + static_cast(syntax), static_cast(match_flags)); + printf(R"( for locale "%s": regex_error: "%s")" + "\n", + locname.c_str(), e.what()); + fixture->fail_regex(); + } + } +}; + +regex_fixture g_regexTester; + +void regex_with_locale_should_throw(const wstring& pattern, const string& locname, const error_type expected, + const syntax_option_type syntax = regex_constants::collate) { + wregex r; + r.imbue(locale{locname}); + try { + r.assign(pattern, syntax); + wprintf(LR"(regex r("%s", 0x%X) succeeded for locale "%hs" (which is bad).)" + L"\n", + pattern.c_str(), static_cast(syntax), locname.c_str()); + g_regexTester.fail_regex(); + } catch (const regex_error& e) { + if (e.code() != expected) { + wprintf(LR"(regex r("%s", 0x%X) with locale "%hs" threw 0x%X; expected 0x%X)" + L"\n", + pattern.c_str(), static_cast(syntax), locname.c_str(), + static_cast(e.code()), static_cast(expected)); + g_regexTester.fail_regex(); + } + } +} + +void test_collating_ranges_german() { + + // special characters in German (umlauts and sharp s) + static constexpr const wchar_t* special_characters[] = { + L"\u00E4", // U+00E4 LATIN SMALL LETTER A WITH DIARESIS + L"\u00C4", // U+00C4 LATIN CAPITAL LETTER A WITH DIARESIS + L"\u00DF", // U+00DF LATIN SMALL LETTER SHARP S + L"\u1E9E", // U+1E9E LATIN CAPITAL LETTER SHARP S + L"\u00F6", // U+00F6 LATIN SMALL LETTER U WITH DIARESIS + L"\u00D6", // U+00D6 LATIN CAPITAL LETTER U WITH DIARESIS + L"\u00FC", // U+00FC LATIN SMALL LETTER O WITH DIARESIS + L"\u00DC" // U+00DC LATIN CAPITAL LETTER O WITH DIARESIS + }; + + // sanity checks: collation not enabled, with or without imbued locale + { + // [a-z], [A-Z] and [A-z] should not match special German characters + for (const wstring& pattern : {L"[a-z]", L"[A-Z]", L"[A-z]"}) { + { + test_wregex nocollate_nolocale(&g_regexTester, pattern); + for (const auto& s : special_characters) { + nocollate_nolocale.should_search_fail(s); + } + } + + { + test_wregex_locale nocollate_locale(&g_regexTester, pattern, "de-DE"); + for (const auto& s : special_characters) { + nocollate_locale.should_search_fail(s); + } + } + } + } + +#ifndef SKIP_COLLATE_TESTS + // de-DE collation order sorts as follows: + // a, A, + // U+00E4 LATIN SMALL LETTER A WITH DIARESIS, + // U+00C4 LATIN CAPITAL LETTER A WITH DIARESIS, + // b, B, ..., o, O, + // U+00F6 LATIN SMALL LETTER O WITH DIARESIS, + // U+00D6 LATIN CAPITAL LETTER O WITH DIARESIS, + // p, P, ..., s, S, + // U+00DF LATIN SMALL LETTER SHARP S + // U+1E9E LATIN CAPITAL LETTER SHARP S, + // t, T, u, U, + // U+00FC LATIN SMALL LETTER U WITH DIARESIS, + // U+00DC LATIN CAPITAL LETTER U WITH DIARESIS, + // v, V, ..., z, Z + + static constexpr const wchar_t* special_characters_without_ae[] = { + L"\u00DF", // U+00DF LATIN SMALL LETTER SHARP S + L"\u1E9E", // U+1E9E LATIN CAPITAL LETTER SHARP S + L"\u00F6", // U+00F6 LATIN SMALL LETTER U WITH DIARESIS + L"\u00D6", // U+00D6 LATIN CAPITAL LETTER U WITH DIARESIS + L"\u00FC", // U+00FC LATIN SMALL LETTER O WITH DIARESIS + L"\u00DC" // U+00DC LATIN CAPITAL LETTER O WITH DIARESIS + }; + + { + test_wregex_locale collate_a_to_a_regex(&g_regexTester, L"[a-a]", "de-DE", regex_constants::collate); + collate_a_to_a_regex.should_search_match(L"a", L"a"); + collate_a_to_a_regex.should_search_fail(L"A"); + collate_a_to_a_regex.should_search_fail(L"\u00E4"); // U+00E4 LATIN SMALL LETTER A WITH DIARESIS + collate_a_to_a_regex.should_search_fail(L"\u00C4"); // U+00C4 LATIN CAPITAL LETTER A WITH DIARESIS + collate_a_to_a_regex.should_search_fail(L"b"); + for (const auto& s : special_characters_without_ae) { + collate_a_to_a_regex.should_search_fail(s); + } + } + + { + test_wregex_locale collate_a_to_A_regex(&g_regexTester, L"[a-A]", "de-DE", regex_constants::collate); + collate_a_to_A_regex.should_search_match(L"a", L"a"); + collate_a_to_A_regex.should_search_match(L"A", L"A"); + collate_a_to_A_regex.should_search_fail(L"\u00E4"); // U+00E4 LATIN SMALL LETTER A WITH DIARESIS + collate_a_to_A_regex.should_search_fail(L"\u00C4"); // U+00C4 LATIN CAPITAL LETTER A WITH DIARESIS + collate_a_to_A_regex.should_search_fail(L"b"); + for (const auto& s : special_characters_without_ae) { + collate_a_to_A_regex.should_search_fail(s); + } + } + + { + test_wregex_locale collate_a_to_ae_regex(&g_regexTester, + L"[a-\u00E4]", // U+00E4 LATIN SMALL LETTER A WITH DIARESIS + "de-DE", regex_constants::collate); + collate_a_to_ae_regex.should_search_match(L"a", L"a"); + collate_a_to_ae_regex.should_search_match(L"A", L"A"); + collate_a_to_ae_regex.should_search_match(L"\u00E4", L"\u00E4"); // U+00E4 LATIN SMALL LETTER A WITH DIARESIS + collate_a_to_ae_regex.should_search_fail(L"\u00C4"); // U+00C4 LATIN CAPITAL LETTER A WITH DIARESIS + collate_a_to_ae_regex.should_search_fail(L"b"); + for (const auto& s : special_characters_without_ae) { + collate_a_to_ae_regex.should_search_fail(s); + } + } + + { + test_wregex_locale collate_a_to_Ae_regex(&g_regexTester, + L"[a-\u00C4]", // U+00C4 LATIN CAPITAL LETTER A WITH DIARESIS + "de-DE", regex_constants::collate); + collate_a_to_Ae_regex.should_search_match(L"a", L"a"); + collate_a_to_Ae_regex.should_search_match(L"A", L"A"); + collate_a_to_Ae_regex.should_search_match(L"\u00E4", L"\u00E4"); // U+00E4 LATIN SMALL LETTER A WITH DIARESIS + collate_a_to_Ae_regex.should_search_match(L"\u00C4", L"\u00C4"); // U+00C4 LATIN CAPITAL LETTER A WITH DIARESIS + collate_a_to_Ae_regex.should_search_fail(L"b"); + for (const auto& s : special_characters_without_ae) { + collate_a_to_Ae_regex.should_search_fail(s); + } + } + + { + test_wregex_locale collate_a_to_b_regex(&g_regexTester, L"[a-b]", "de-DE", regex_constants::collate); + collate_a_to_b_regex.should_search_match(L"a", L"a"); + collate_a_to_b_regex.should_search_match(L"A", L"A"); + collate_a_to_b_regex.should_search_match(L"\u00E4", L"\u00E4"); // U+00E4 LATIN SMALL LETTER A WITH DIARESIS + collate_a_to_b_regex.should_search_match(L"\u00C4", L"\u00C4"); // U+00C4 LATIN CAPITAL LETTER A WITH DIARESIS + collate_a_to_b_regex.should_search_match(L"b", L"b"); + for (const auto& s : special_characters_without_ae) { + collate_a_to_b_regex.should_search_fail(s); + } + } + + static constexpr const wchar_t* special_characters_without_sharp_s[] = { + L"\u00E4", // U+00E4 LATIN SMALL LETTER A WITH DIARESIS + L"\u00C4", // U+00C4 LATIN CAPITAL LETTER A WITH DIARESIS + L"\u00F6", // U+00F6 LATIN SMALL LETTER U WITH DIARESIS + L"\u00D6", // U+00D6 LATIN CAPITAL LETTER U WITH DIARESIS + L"\u00FC", // U+00FC LATIN SMALL LETTER O WITH DIARESIS + L"\u00DC" // U+00DC LATIN CAPITAL LETTER O WITH DIARESIS + }; + + { + test_wregex_locale collate_s_to_S_regex(&g_regexTester, L"[s-S]", "de-DE", regex_constants::collate); + collate_s_to_S_regex.should_search_fail(L"r"); + collate_s_to_S_regex.should_search_match(L"s", L"s"); + collate_s_to_S_regex.should_search_match(L"S", L"S"); + collate_s_to_S_regex.should_search_fail(L"\u00DF"); // U+00DF LATIN SMALL LETTER SHARP S + collate_s_to_S_regex.should_search_fail(L"\u1E9E"); // U+1E9E LATIN CAPITAL LETTER SHARP S + collate_s_to_S_regex.should_search_fail(L"t"); + collate_s_to_S_regex.should_search_fail(L"u"); + for (const auto& s : special_characters_without_sharp_s) { + collate_s_to_S_regex.should_search_fail(s); + } + } + + { + test_wregex_locale collate_s_to_sharp_s_regex(&g_regexTester, + L"[s-\u00DF]", // U+00DF LATIN SMALL LETTER SHARP S + "de-DE", regex_constants::collate); + collate_s_to_sharp_s_regex.should_search_fail(L"r"); + collate_s_to_sharp_s_regex.should_search_match(L"s", L"s"); + collate_s_to_sharp_s_regex.should_search_match(L"S", L"S"); + collate_s_to_sharp_s_regex.should_search_match(L"\u00DF", L"\u00DF"); // U+00DF LATIN SMALL LETTER SHARP S + collate_s_to_sharp_s_regex.should_search_fail(L"\u1E9E"); // U+1E9E LATIN CAPITAL LETTER SHARP S + collate_s_to_sharp_s_regex.should_search_fail(L"t"); + collate_s_to_sharp_s_regex.should_search_fail(L"u"); + for (const auto& s : special_characters_without_sharp_s) { + collate_s_to_sharp_s_regex.should_search_fail(s); + } + } + + { + test_wregex_locale collate_s_to_Sharp_S_regex(&g_regexTester, L"[s-\u1E9E]", // LATIN CAPITAL LETTER SHARP S + "de-DE", regex_constants::collate); + collate_s_to_Sharp_S_regex.should_search_fail(L"r"); + collate_s_to_Sharp_S_regex.should_search_match(L"s", L"s"); + collate_s_to_Sharp_S_regex.should_search_match(L"S", L"S"); + collate_s_to_Sharp_S_regex.should_search_match(L"\u00DF", L"\u00DF"); // U+00DF LATIN SMALL LETTER SHARP S + collate_s_to_Sharp_S_regex.should_search_match(L"\u1E9E", L"\u1E9E"); // U+1E9E LATIN CAPITAL LETTER SHARP S + collate_s_to_Sharp_S_regex.should_search_fail(L"t"); + collate_s_to_Sharp_S_regex.should_search_fail(L"u"); + for (const auto& s : special_characters_without_sharp_s) { + collate_s_to_Sharp_S_regex.should_search_fail(s); + } + } + + { + test_wregex_locale collate_s_to_t_regex(&g_regexTester, L"[s-t]", "de-DE", regex_constants::collate); + collate_s_to_t_regex.should_search_fail(L"r"); + collate_s_to_t_regex.should_search_match(L"s", L"s"); + collate_s_to_t_regex.should_search_match(L"S", L"S"); + collate_s_to_t_regex.should_search_match(L"\u00DF", L"\u00DF"); // U+00DF LATIN SMALL LETTER SHARP S + collate_s_to_t_regex.should_search_match(L"\u1E9E", L"\u1E9E"); // U+1E9E LATIN CAPITAL LETTER SHARP S + collate_s_to_t_regex.should_search_match(L"t", L"t"); + collate_s_to_t_regex.should_search_fail(L"u"); + for (const auto& s : special_characters_without_sharp_s) { + collate_s_to_t_regex.should_search_fail(s); + } + } + + { + test_wregex_locale collate_A_to_z_regex(&g_regexTester, L"[A-z]", "de-DE", regex_constants::collate); + collate_A_to_z_regex.should_search_fail(L"a"); + for (wchar_t ascii_upper = L'A'; ascii_upper < L'Z'; ++ascii_upper) { + collate_A_to_z_regex.should_search_match(wstring(1, ascii_upper), wstring(1, ascii_upper)); + } + for (wchar_t ascii_lower = L'b'; ascii_lower <= L'z'; ++ascii_lower) { + collate_A_to_z_regex.should_search_match(wstring(1, ascii_lower), wstring(1, ascii_lower)); + } + for (const auto& s : special_characters) { + collate_A_to_z_regex.should_search_match(s, s); + } + collate_A_to_z_regex.should_search_fail(L"Z"); + } + + regex_with_locale_should_throw(L"[A-a]", "de-DE", error_range); + regex_with_locale_should_throw(L"[\u00DF-S]", // U+00DF LATIN SMALL LETTER SHARP S + "de-DE", error_range); + regex_with_locale_should_throw( + L"[\u1E9E-\u00DF]", // U+1E9E LATIN CAPITAL LETTER SHARP S, U+00DF LATIN SMALL LETTER SHARP S + "de-DE", error_range); +#endif // !defined(SKIP_COLLATE_TESTS) +} + +int main() { + test_collating_ranges_german(); + + return g_regexTester.result(); +} From 43dd4e4d1b08bf259dee8c8aeeb5f5f23df289e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Tue, 25 Mar 2025 04:18:16 +0100 Subject: [PATCH 10/14] ``: Clean up parsing logic for quantifiers (#5253) Co-authored-by: Stephan T. Lavavej --- stl/inc/regex | 33 ++++++++----------- .../std/tests/VSO_0000000_regex_use/test.cpp | 21 ++++++++++++ 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index 1393336ac78..17fd2ee72c5 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -1202,8 +1202,7 @@ enum _Node_flags : int { // flags for nfa nodes with special properties _Fl_none = 0x00, _Fl_negate = 0x01, _Fl_greedy = 0x02, - _Fl_final = 0x04, - _Fl_longest = 0x08 + _Fl_longest = 0x08 // TRANSITION, ABI: 0x04 is unused; the parser previously marked some nodes with it }; _BITMASK_OPS(_EMPTY_ARGUMENT, _Node_flags) @@ -1521,7 +1520,7 @@ public: void _Add_eol(); void _Add_wbound(); void _Add_dot(); - void _Add_char(_Elem _Ch); + void _Add_char2(_Elem _Ch); void _Add_class(); void _Add_char_to_class(_Elem _Ch); void _Add_range2(_Elem, _Elem); @@ -1536,9 +1535,8 @@ public: void _Add_backreference(unsigned int _Idx); _Node_base* _Begin_if(_Node_base* _Start); void _Else_if(_Node_base*, _Node_base*); - void _Add_rep(int _Min, int _Max, bool _Greedy); + void _Add_rep2(int _Min, int _Max, bool _Greedy); void _Negate(); - void _Mark_final(); _Root_node* _End_pattern(); private: @@ -2773,11 +2771,6 @@ void _Builder<_FwdIt, _Elem, _RxTraits>::_Negate() { // set flag _Current->_Flags ^= _Fl_negate; } -template -void _Builder<_FwdIt, _Elem, _RxTraits>::_Mark_final() { // set flag - _Current->_Flags |= _Fl_final; -} - template _Node_base* _Builder<_FwdIt, _Elem, _RxTraits>::_Getmark() const { return _Current; @@ -2840,8 +2833,8 @@ void _Builder<_FwdIt, _Elem, _RxTraits>::_Add_str_node() { // add string node } template -void _Builder<_FwdIt, _Elem, _RxTraits>::_Add_char(_Elem _Ch) { // append character - if (_Current->_Kind != _N_str || (_Current->_Flags & _Fl_final)) { +void _Builder<_FwdIt, _Elem, _RxTraits>::_Add_char2(_Elem _Ch) { // append character + if (_Current->_Kind != _N_str) { _Add_str_node(); } @@ -3090,11 +3083,12 @@ void _Builder<_FwdIt, _Elem, _RxTraits>::_Else_if(_Node_base* _Start, _Node_base } template -void _Builder<_FwdIt, _Elem, _RxTraits>::_Add_rep(int _Min, int _Max, bool _Greedy) { // add repeat node +void _Builder<_FwdIt, _Elem, _RxTraits>::_Add_rep2(int _Min, int _Max, bool _Greedy) { // add repeat node if (_Current->_Kind == _N_str && static_cast<_Node_str<_Elem>*>(_Current)->_Data._Size() != 1) { // move final character to new string node _Node_str<_Elem>* _Node = static_cast<_Node_str<_Elem>*>(_Current); - _Add_char(_Node->_Data._Del()); + _Add_str_node(); + _Add_char2(_Node->_Data._Del()); } _Node_base* _Pos = _Current; @@ -4419,7 +4413,7 @@ void _Parser<_FwdIt, _Elem, _RxTraits>::_AtomEscape() { // check for valid atom if (!(_L_flags & _L_bzr_chr)) { _Error(regex_constants::error_escape); } else { - _Nfa._Add_char(static_cast<_Elem>(_Val)); + _Nfa._Add_char2(static_cast<_Elem>(_Val)); } } else if (_Grp_idx < static_cast(_Val) || _Finished_grps.size() <= static_cast(_Val) || !_Finished_grps[static_cast(_Val)]) { @@ -4428,7 +4422,7 @@ void _Parser<_FwdIt, _Elem, _RxTraits>::_AtomEscape() { // check for valid atom _Nfa._Add_backreference(static_cast(_Val)); } } else if (_CharacterEscape()) { - _Nfa._Add_char(static_cast<_Elem>(_Val)); + _Nfa._Add_char2(static_cast<_Elem>(_Val)); } else if (!(_L_flags & _L_esc_wsd) || !_CharacterClassEscape(true)) { _Error(regex_constants::error_escape); } @@ -4471,14 +4465,13 @@ void _Parser<_FwdIt, _Elem, _RxTraits>::_Quantifier() { // check for quantifier } } - _Nfa._Mark_final(); _Next(); const bool _Greedy = !(_L_flags & _L_ngr_rep) || _Mchar != _Meta_query; if (!_Greedy) { // add non-greedy repeat node _Next(); } - _Nfa._Add_rep(_Min, _Max, _Greedy); + _Nfa._Add_rep2(_Min, _Max, _Greedy); } template @@ -4519,7 +4512,7 @@ bool _Parser<_FwdIt, _Elem, _RxTraits>::_Alternative() { // check for valid alte _Nfa._Add_bol(); _Next(); if ((_L_flags & _L_star_beg) && _Mchar == _Meta_star && !_Found) { - _Nfa._Add_char(_Char); + _Nfa._Add_char2(_Char); _Next(); } else { _Quant = false; @@ -4536,7 +4529,7 @@ bool _Parser<_FwdIt, _Elem, _RxTraits>::_Alternative() { // check for valid alte } else if (_Mchar == _Meta_rsq && !(_L_flags & _L_paren_bal)) { _Error(regex_constants::error_brack); } else { // add character - _Nfa._Add_char(_Char); + _Nfa._Add_char2(_Char); _Next(); } diff --git a/tests/std/tests/VSO_0000000_regex_use/test.cpp b/tests/std/tests/VSO_0000000_regex_use/test.cpp index a98b6ff9122..ba9c575940a 100644 --- a/tests/std/tests/VSO_0000000_regex_use/test.cpp +++ b/tests/std/tests/VSO_0000000_regex_use/test.cpp @@ -1151,6 +1151,26 @@ void test_gh_5214() { } } +void test_gh_5253() { + // GH-5253 cleaned up parsing logic for quantifiers that were applied to single characters + g_regexTester.should_match("abbb", "ab*"); + g_regexTester.should_not_match("abab", "ab*"); + g_regexTester.should_match("abbb", "(a)b*"); + g_regexTester.should_not_match("abab", "(a)b*"); + g_regexTester.should_match("abbb", "a(b)*"); + g_regexTester.should_not_match("abab", "a(b)*"); + g_regexTester.should_match("abbb", "(a)(b)*"); + g_regexTester.should_not_match("abab", "(a)(b)*"); + g_regexTester.should_not_match("abbb", "(ab)*"); + g_regexTester.should_match("abab", "(ab)*"); + g_regexTester.should_not_match("abbb", "(?:ab)*"); + g_regexTester.should_match("abab", "(?:ab)*"); + g_regexTester.should_match("aaaa", "a*"); + g_regexTester.should_not_match("b", "a*"); + g_regexTester.should_match("", "()*"); + g_regexTester.should_not_match("a", "()*"); +} + int main() { test_dev10_449367_case_insensitivity_should_work(); test_dev11_462743_regex_collate_should_not_disable_regex_icase(); @@ -1187,6 +1207,7 @@ int main() { test_gh_5167(); test_gh_5192(); test_gh_5214(); + test_gh_5253(); return g_regexTester.result(); } From b8e575bb0e509a4ddfa732dc2cf04d0bad094920 Mon Sep 17 00:00:00 2001 From: "A. Jiang" Date: Tue, 25 Mar 2025 11:23:07 +0800 Subject: [PATCH 11/14] Initially use `[[msvc::no_unique_address]]` for some C++23 components (#4960) Co-authored-by: Stephan T. Lavavej --- stl/inc/algorithm | 4 +- stl/inc/mdspan | 114 +++++++++--------- stl/inc/ranges | 21 ++-- stl/inc/xmemory | 11 +- stl/inc/xutility | 4 +- stl/inc/yvals_core.h | 12 ++ tests/std/include/test_mdspan_support.hpp | 22 ++++ tests/std/test.lst | 2 + .../env.lst | 4 + .../test.compile.pass.cpp | 102 ++++++++++++++++ .../tests/P0009R18_mdspan_extents/test.cpp | 6 - .../P0009R18_mdspan_layout_left/test.cpp | 6 - .../P0009R18_mdspan_layout_right/test.cpp | 6 - .../P0009R18_mdspan_layout_stride/test.cpp | 6 - .../std/tests/P0009R18_mdspan_mdspan/test.cpp | 41 ------- tests/std/tests/P0009R18_mdspan_msabi/env.lst | 4 + .../test.compile.pass.cpp | 54 +++++++++ 17 files changed, 284 insertions(+), 135 deletions(-) create mode 100644 tests/std/tests/GH_001394_msvc_no_unique_address_23/env.lst create mode 100644 tests/std/tests/GH_001394_msvc_no_unique_address_23/test.compile.pass.cpp create mode 100644 tests/std/tests/P0009R18_mdspan_msabi/env.lst create mode 100644 tests/std/tests/P0009R18_mdspan_msabi/test.compile.pass.cpp diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 04a8a9d7496..bfeed4d3cf1 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -398,8 +398,8 @@ namespace ranges { #if _HAS_CXX23 _EXPORT_STD template struct in_value_result { - /* [[no_unique_address]] */ _In in; - /* [[no_unique_address]] */ _Ty value; + _MSVC_NO_UNIQUE_ADDRESS _In in; + _MSVC_NO_UNIQUE_ADDRESS _Ty value; template requires convertible_to && convertible_to diff --git a/stl/inc/mdspan b/stl/inc/mdspan index bedc6046035..fa151c8a499 100644 --- a/stl/inc/mdspan +++ b/stl/inc/mdspan @@ -23,8 +23,8 @@ _STL_DISABLE_CLANG_WARNINGS #undef new // TRANSITION, non-_Ugly attribute tokens -#pragma push_macro("empty_bases") -#undef empty_bases +#pragma push_macro("msvc") +#undef msvc _STD_BEGIN template @@ -1073,37 +1073,37 @@ concept _Elidable_layout_mapping = || (same_as<_LayoutPolicy, layout_stride> && _Extents::rank() == 0); template -struct _Mdspan_mapping_base { +struct _Mdspan_mapping_holder { _STL_INTERNAL_STATIC_ASSERT(_Is_extents<_Extents>); using _Mapping = _LayoutPolicy::template mapping<_Extents>; - constexpr _Mdspan_mapping_base() noexcept = default; + constexpr _Mdspan_mapping_holder() noexcept = default; - constexpr explicit _Mdspan_mapping_base(const _Extents& _Exts) : _Map(_Exts) {} + constexpr explicit _Mdspan_mapping_holder(const _Extents& _Exts) : _Map(_Exts) {} - constexpr explicit _Mdspan_mapping_base(_Extents&& _Exts) : _Map(_STD move(_Exts)) {} + constexpr explicit _Mdspan_mapping_holder(_Extents&& _Exts) : _Map(_STD move(_Exts)) {} template - constexpr explicit _Mdspan_mapping_base(const _OtherMapping& _Map_) : _Map(_Map_) {} + constexpr explicit _Mdspan_mapping_holder(const _OtherMapping& _Map_) : _Map(_Map_) {} - _Mapping _Map = _Mapping(); + _MSVC_NO_UNIQUE_ADDRESS _Mapping _Map = _Mapping(); }; template _LayoutPolicy> -struct _Mdspan_mapping_base<_Extents, _LayoutPolicy> { +struct _Mdspan_mapping_holder<_Extents, _LayoutPolicy> { _STL_INTERNAL_STATIC_ASSERT(_Is_extents<_Extents>); using _Mapping = _LayoutPolicy::template mapping<_Extents>; - constexpr _Mdspan_mapping_base() noexcept = default; + constexpr _Mdspan_mapping_holder() noexcept = default; - constexpr explicit _Mdspan_mapping_base(const _Extents&) noexcept {} + constexpr explicit _Mdspan_mapping_holder(const _Extents&) noexcept {} - constexpr explicit _Mdspan_mapping_base(_Extents&&) noexcept {} + constexpr explicit _Mdspan_mapping_holder(_Extents&&) noexcept {} template - constexpr explicit _Mdspan_mapping_base(const _OtherMapping& _Map_) { + constexpr explicit _Mdspan_mapping_holder(const _OtherMapping& _Map_) { // NB: Constructing _Mapping from _OtherMapping may have side effects - we should create a temporary. if constexpr (!_Elidable_layout_mapping) { (void) _Mapping{_Map_}; @@ -1117,21 +1117,21 @@ template concept _Elidable_accessor_policy = _Is_specialization_v<_AccessorPolicy, default_accessor>; template -struct _Mdspan_accessor_base { - constexpr _Mdspan_accessor_base() noexcept = default; +struct _Mdspan_accessor_holder { + constexpr _Mdspan_accessor_holder() noexcept = default; template - constexpr explicit _Mdspan_accessor_base(const _OtherAccessorPolicy& _Acc_) : _Acc(_Acc_) {} + constexpr explicit _Mdspan_accessor_holder(const _OtherAccessorPolicy& _Acc_) : _Acc(_Acc_) {} - _AccessorPolicy _Acc = _AccessorPolicy(); + _MSVC_NO_UNIQUE_ADDRESS _AccessorPolicy _Acc = _AccessorPolicy(); }; template <_Elidable_accessor_policy _AccessorPolicy> -struct _Mdspan_accessor_base<_AccessorPolicy> { - constexpr _Mdspan_accessor_base() noexcept = default; +struct _Mdspan_accessor_holder<_AccessorPolicy> { + constexpr _Mdspan_accessor_holder() noexcept = default; template - constexpr explicit _Mdspan_accessor_base(const _OtherAccessorPolicy& _Acc_) { + constexpr explicit _Mdspan_accessor_holder(const _OtherAccessorPolicy& _Acc_) { // NB: Constructing _AccessorPolicy from _OtherAccessorPolicy may have side effects - we should create a // temporary. if constexpr (!_Elidable_accessor_policy<_OtherAccessorPolicy>) { @@ -1160,8 +1160,7 @@ _NODISCARD constexpr _IndexType _Mdspan_checked_index_cast(_OtherIndexType&& _Id _EXPORT_STD template > -class __declspec(empty_bases) mdspan : private _Mdspan_mapping_base<_Extents, _LayoutPolicy>, - private _Mdspan_accessor_base<_AccessorPolicy> { +class mdspan { public: using extents_type = _Extents; using layout_type = _LayoutPolicy; @@ -1176,9 +1175,6 @@ public: using reference = accessor_type::reference; private: - using _Mapping_base = _Mdspan_mapping_base; - using _Accessor_base = _Mdspan_accessor_base; - static_assert( sizeof(element_type) > 0, "ElementType must be a complete type (N4950 [mdspan.mdspan.overview]/2.1)."); static_assert( @@ -1208,7 +1204,7 @@ public: } _NODISCARD constexpr index_type extent(_In_range_(<, extents_type::_Rank) const rank_type _Idx) const noexcept { - return this->_Map.extents().extent(_Idx); + return _Mapping._Map.extents().extent(_Idx); } constexpr mdspan() noexcept(is_nothrow_default_constructible_v @@ -1229,8 +1225,7 @@ public: constexpr explicit mdspan(data_handle_type _Ptr_, _OtherIndexTypes... _Exts) noexcept(is_nothrow_constructible_v && is_nothrow_default_constructible_v) // strengthened - : _Mapping_base(extents_type{static_cast(_STD move(_Exts))...}), _Accessor_base(), - _Ptr(_STD move(_Ptr_)) {} + : _Mapping(extents_type{static_cast(_STD move(_Exts))...}), _Accessor(), _Ptr(_STD move(_Ptr_)) {} template requires is_convertible_v @@ -1240,7 +1235,7 @@ public: constexpr explicit(_Size != rank_dynamic()) mdspan(data_handle_type _Ptr_, span<_OtherIndexType, _Size> _Exts) noexcept(is_nothrow_constructible_v && is_nothrow_default_constructible_v) // strengthened - : _Mapping_base(extents_type{_Exts}), _Accessor_base(), _Ptr(_STD move(_Ptr_)) {} + : _Mapping(extents_type{_Exts}), _Accessor(), _Ptr(_STD move(_Ptr_)) {} template requires is_convertible_v @@ -1251,23 +1246,23 @@ public: mdspan(data_handle_type _Ptr_, const array<_OtherIndexType, _Size>& _Exts) noexcept(is_nothrow_constructible_v && is_nothrow_default_constructible_v) // strengthened - : _Mapping_base(extents_type{_Exts}), _Accessor_base(), _Ptr(_STD move(_Ptr_)) {} + : _Mapping(extents_type{_Exts}), _Accessor(), _Ptr(_STD move(_Ptr_)) {} constexpr mdspan(data_handle_type _Ptr_, const extents_type& _Exts) noexcept(is_nothrow_constructible_v && is_nothrow_default_constructible_v) // strengthened requires is_constructible_v && is_default_constructible_v - : _Mapping_base(_Exts), _Accessor_base(), _Ptr(_STD move(_Ptr_)) {} + : _Mapping(_Exts), _Accessor(), _Ptr(_STD move(_Ptr_)) {} constexpr mdspan(data_handle_type _Ptr_, const mapping_type& _Map_) noexcept(is_nothrow_copy_constructible_v && is_nothrow_default_constructible_v) // strengthened requires is_default_constructible_v - : _Mapping_base(_Map_), _Accessor_base(), _Ptr(_STD move(_Ptr_)) {} + : _Mapping(_Map_), _Accessor(), _Ptr(_STD move(_Ptr_)) {} constexpr mdspan(data_handle_type _Ptr_, const mapping_type& _Map_, const accessor_type& _Acc_) noexcept( is_nothrow_copy_constructible_v && is_nothrow_copy_constructible_v) // strengthened - : _Mapping_base(_Map_), _Accessor_base(_Acc_), _Ptr(_STD move(_Ptr_)) {} + : _Mapping(_Map_), _Accessor(_Acc_), _Ptr(_STD move(_Ptr_)) {} template requires is_constructible_v&> @@ -1280,7 +1275,7 @@ public: && is_nothrow_constructible_v&> && is_nothrow_constructible_v) // strengthened - : _Mapping_base(_Other.mapping()), _Accessor_base(_Other.accessor()), _Ptr(_Other.data_handle()) { + : _Mapping(_Other.mapping()), _Accessor(_Other.accessor()), _Ptr(_Other.data_handle()) { static_assert(is_constructible_v, "The data_handle_type must be constructible from const typename OtherAccessor::data_handle_type& (N4950 " "[mdspan.mdspan.cons]/20.1)."); @@ -1346,20 +1341,21 @@ public: _NODISCARD constexpr size_type size() const noexcept { #if _ITERATOR_DEBUG_LEVEL != 0 if constexpr (rank_dynamic() != 0) { - _STL_VERIFY(this->_Map.extents().template _Is_dynamic_multidim_index_space_size_representable(), + _STL_VERIFY( + _Mapping._Map.extents().template _Is_dynamic_multidim_index_space_size_representable(), "The size of the multidimensional index space extents() must be representable as a value of type " "size_type (N4950 [mdspan.mdspan.members]/7)."); } #endif // _ITERATOR_DEBUG_LEVEL != 0 return static_cast( - _Fwd_prod_of_extents::_Calculate(this->_Map.extents(), extents_type::_Rank)); + _Fwd_prod_of_extents::_Calculate(_Mapping._Map.extents(), extents_type::_Rank)); } _NODISCARD constexpr bool empty() const noexcept { if constexpr (extents_type::_Multidim_index_space_size_is_always_zero) { return true; } else { - const extents_type& _Exts = this->_Map.extents(); + const extents_type& _Exts = _Mapping._Map.extents(); for (rank_type _Idx = 0; _Idx < extents_type::_Rank; ++_Idx) { if (_Exts.extent(_Idx) == 0) { return true; @@ -1373,16 +1369,16 @@ public: swap(_Left._Ptr, _Right._Ptr); // intentional ADL if constexpr (!_Elidable_layout_mapping) { - swap(_Left._Map, _Right._Map); // intentional ADL + swap(_Left._Mapping._Map, _Right._Mapping._Map); // intentional ADL } if constexpr (!_Elidable_accessor_policy) { - swap(_Left._Acc, _Right._Acc); // intentional ADL + swap(_Left._Accessor._Acc, _Right._Accessor._Acc); // intentional ADL } } _NODISCARD constexpr const extents_type& extents() const noexcept { - return this->_Map.extents(); + return _Mapping._Map.extents(); } _NODISCARD constexpr const data_handle_type& data_handle() const noexcept { @@ -1390,11 +1386,11 @@ public: } _NODISCARD constexpr const mapping_type& mapping() const noexcept { - return this->_Map; + return _Mapping._Map; } _NODISCARD constexpr const accessor_type& accessor() const noexcept { - return this->_Acc; + return _Accessor._Acc; } _NODISCARD static constexpr bool is_always_unique() noexcept /* strengthened */ { @@ -1412,38 +1408,42 @@ public: return _Result; } - _NODISCARD constexpr bool is_unique() const noexcept(noexcept(this->_Map.is_unique())) /* strengthened */ { - return this->_Map.is_unique(); + _NODISCARD constexpr bool is_unique() const noexcept(noexcept(_Mapping._Map.is_unique())) /* strengthened */ { + return _Mapping._Map.is_unique(); } - _NODISCARD constexpr bool is_exhaustive() const noexcept(noexcept(this->_Map.is_exhaustive())) /* strengthened */ { - return this->_Map.is_exhaustive(); + _NODISCARD constexpr bool is_exhaustive() const + noexcept(noexcept(_Mapping._Map.is_exhaustive())) /* strengthened */ { + return _Mapping._Map.is_exhaustive(); } - _NODISCARD constexpr bool is_strided() const noexcept(noexcept(this->_Map.is_strided())) /* strengthened */ { - return this->_Map.is_strided(); + _NODISCARD constexpr bool is_strided() const noexcept(noexcept(_Mapping._Map.is_strided())) /* strengthened */ { + return _Mapping._Map.is_strided(); } _NODISCARD constexpr index_type stride(const rank_type _Idx) const - noexcept(noexcept(this->_Map.stride(_Idx))) /* strengthened */ { - return this->_Map.stride(_Idx); + noexcept(noexcept(_Mapping._Map.stride(_Idx))) /* strengthened */ { + return _Mapping._Map.stride(_Idx); } private: template _NODISCARD constexpr reference _Access_impl(_OtherIndexTypes... _Indices) const - noexcept(noexcept(this->_Acc.access(_Ptr, static_cast(this->_Map(_Indices...))))) { + noexcept(noexcept(_Accessor._Acc.access(_Ptr, static_cast(_Mapping._Map(_Indices...))))) { _STL_INTERNAL_STATIC_ASSERT(conjunction_v...>); #if _MSVC_STL_HARDENING_MDSPAN || _ITERATOR_DEBUG_LEVEL != 0 - _STL_VERIFY(this->_Map.extents()._Contains_multidimensional_index(make_index_sequence{}, _Indices...), - "mdspan subscript out of range; extents_type::index-cast(std::move(indices)) must be " - "a multidimensional index in extents() (N5001 [mdspan.mdspan.members]/3)."); + _STL_VERIFY( + _Mapping._Map.extents()._Contains_multidimensional_index(make_index_sequence{}, _Indices...), + "mdspan subscript out of range; extents_type::index-cast(std::move(indices)) must be a multidimensional " + "index in extents() (N5001 [mdspan.mdspan.members]/3)."); #endif - return this->_Acc.access(_Ptr, static_cast(this->_Map(_Indices...))); + return _Accessor._Acc.access(_Ptr, static_cast(_Mapping._Map(_Indices...))); } - /* [[no_unique_address]] */ data_handle_type _Ptr = data_handle_type(); + _MSVC_NO_UNIQUE_ADDRESS _Mdspan_mapping_holder _Mapping; + _MSVC_NO_UNIQUE_ADDRESS _Mdspan_accessor_holder _Accessor; + _MSVC_NO_UNIQUE_ADDRESS data_handle_type _Ptr = data_handle_type(); }; template @@ -1480,7 +1480,7 @@ mdspan(const typename _AccessorType::data_handle_type&, const _MappingType&, con _STD_END // TRANSITION, non-_Ugly attribute tokens -#pragma pop_macro("empty_bases") +#pragma pop_macro("msvc") #pragma pop_macro("new") _STL_RESTORE_CLANG_WARNINGS diff --git a/stl/inc/ranges b/stl/inc/ranges index 06520461db3..8768eede7ea 100644 --- a/stl/inc/ranges +++ b/stl/inc/ranges @@ -31,6 +31,10 @@ _STL_DISABLE_CLANG_WARNINGS #pragma push_macro("new") #undef new +// TRANSITION, non-_Ugly attribute tokens +#pragma push_macro("msvc") +#undef msvc + _STD_BEGIN namespace ranges { // MUCH machinery defined in , some in <__msvc_ranges_to.hpp> @@ -1087,7 +1091,7 @@ namespace ranges { using _Index_type = conditional_t, ptrdiff_t, _Bo>; const _Ty* _Value{}; - /* [[no_unique_address]] */ _Index_type _Current{}; + _Index_type _Current{}; constexpr explicit _Iterator(const _Ty* _Val, _Index_type _Bo_ = _Index_type{}) noexcept // strengthened : _Value(_Val), _Current(_Bo_) { @@ -3453,7 +3457,7 @@ namespace ranges { using _Parent_t = _Maybe_const<_Const, join_with_view>; using _Base = _Maybe_const<_Const, _Vw>; - /* [[no_unique_address]] */ sentinel_t<_Base> _Last{}; + _MSVC_NO_UNIQUE_ADDRESS sentinel_t<_Base> _Last{}; constexpr explicit _Sentinel(_Parent_t& _Parent) noexcept(noexcept(_RANGES end(_Parent._Range)) @@ -5129,7 +5133,7 @@ namespace ranges { using _Base_iterator = iterator_t<_Base_t>; using _Reference_type = tuple, range_reference_t<_Base_t>>; - /* [[no_unique_address]] */ _Base_iterator _Current{}; + _MSVC_NO_UNIQUE_ADDRESS _Base_iterator _Current{}; range_difference_t<_Base_t> _Pos = 0; constexpr explicit _Iterator(_Base_iterator _Current_, range_difference_t<_Base_t> _Pos_) @@ -5289,7 +5293,7 @@ namespace ranges { using _Base_t = _Maybe_const<_Const, _Vw>; using _Base_sentinel = sentinel_t<_Base_t>; - /* [[no_unique_address]] */ _Base_sentinel _End{}; + _MSVC_NO_UNIQUE_ADDRESS _Base_sentinel _End{}; constexpr explicit _Sentinel(_Base_sentinel _End_) noexcept(is_nothrow_move_constructible_v<_Base_sentinel>) // strengthened @@ -5725,8 +5729,8 @@ namespace ranges { using _Base_iterator = iterator_t<_Base>; using _Base_sentinel = sentinel_t<_Base>; - /* [[no_unique_address]] */ _Base_iterator _Current{}; - /* [[no_unique_address]] */ _Base_sentinel _End{}; + _MSVC_NO_UNIQUE_ADDRESS _Base_iterator _Current{}; + _MSVC_NO_UNIQUE_ADDRESS _Base_sentinel _End{}; range_difference_t<_Base> _Count = 0; range_difference_t<_Base> _Missing = 0; @@ -6284,7 +6288,7 @@ namespace ranges { class _Sentinel { private: friend slide_view; - /* [[no_unique_address]] */ sentinel_t<_Vw> _Last{}; + _MSVC_NO_UNIQUE_ADDRESS sentinel_t<_Vw> _Last{}; constexpr explicit _Sentinel(sentinel_t<_Vw> _Last_) noexcept(is_nothrow_move_constructible_v>) /* strengthened */ @@ -9243,6 +9247,9 @@ _EXPORT_STD namespace views = ranges::views; _STD_END +// TRANSITION, non-_Ugly attribute tokens +#pragma pop_macro("msvc") + #pragma pop_macro("new") _STL_RESTORE_CLANG_WARNINGS #pragma warning(pop) diff --git a/stl/inc/xmemory b/stl/inc/xmemory index f65246466cc..80041fc71bc 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -25,6 +25,10 @@ _STL_DISABLE_CLANG_WARNINGS #pragma push_macro("new") #undef new +// TRANSITION, non-_Ugly attribute tokens +#pragma push_macro("msvc") +#undef msvc + #if _USE_STD_VECTOR_ALGORITHMS extern "C" { void* __stdcall __std_remove_1(void* _First, void* _Last, uint8_t _Val) noexcept; @@ -2736,8 +2740,8 @@ namespace ranges { _EXPORT_STD template #endif // ^^^ !defined(__cpp_lib_byte) ^^^ struct elements_of { - /* [[no_unique_address]] */ _Rng range; - /* [[no_unique_address]] */ _Alloc allocator{}; + _MSVC_NO_UNIQUE_ADDRESS _Rng range; + _MSVC_NO_UNIQUE_ADDRESS _Alloc allocator{}; }; #if defined(__cpp_lib_byte) @@ -2801,6 +2805,9 @@ _STD_END #define _LIST_REMOVE_RETURN void #endif // ^^^ !_HAS_CXX20 ^^^ +// TRANSITION, non-_Ugly attribute tokens +#pragma pop_macro("msvc") + #pragma pop_macro("new") _STL_RESTORE_CLANG_WARNINGS #pragma warning(pop) diff --git a/stl/inc/xutility b/stl/inc/xutility index 222054fefaf..fbce0c9df7c 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -4972,8 +4972,8 @@ namespace ranges { #if _HAS_CXX23 _EXPORT_STD template struct out_value_result { - /* [[no_unique_address]] */ _Out out; - /* [[no_unique_address]] */ _Ty value; + _MSVC_NO_UNIQUE_ADDRESS _Out out; + _MSVC_NO_UNIQUE_ADDRESS _Ty value; template <_Convertible_from _OOut, _Convertible_from _TTy> constexpr operator out_value_result<_OOut, _TTy>() const& { diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index de4a5eb3957..2e1f7472d11 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -730,6 +730,18 @@ #define _MSVC_LIFETIMEBOUND #endif +#if _HAS_CXX23 // TRANSITION, ABI, should just use [[no_unique_address]] when _HAS_CXX20. +// Should we enable use of [[msvc::no_unique_address]] or [[no_unique_address]] to allow potentially-overlapping member +// subobjects? +#if _HAS_MSVC_ATTRIBUTE(no_unique_address) +#define _MSVC_NO_UNIQUE_ADDRESS [[msvc::no_unique_address]] +#elif __has_cpp_attribute(no_unique_address) // TRANSITION, DevCom-10747012, EDG recognizes [[no_unique_address]]. +#define _MSVC_NO_UNIQUE_ADDRESS [[no_unique_address]] +#else +#error Either [[msvc::no_unique_address]] or [[no_unique_address]] must be supported because this is ABI-critical. +#endif +#endif // _HAS_CXX23 + #undef _HAS_MSVC_ATTRIBUTE #pragma pop_macro("lifetimebound") #pragma pop_macro("intrinsic") diff --git a/tests/std/include/test_mdspan_support.hpp b/tests/std/include/test_mdspan_support.hpp index 500d22be37a..e9706697bed 100644 --- a/tests/std/include/test_mdspan_support.hpp +++ b/tests/std/include/test_mdspan_support.hpp @@ -175,6 +175,28 @@ constexpr bool check_accessor_policy_requirements() { return true; } +template +struct TrivialAccessor { + using offset_policy = TrivialAccessor; + using element_type = ElementType; + using reference = ElementType&; + using data_handle_type = ElementType*; + + constexpr reference access(data_handle_type handle, std::size_t off) const noexcept { + return handle[off]; + } + + constexpr data_handle_type offset(data_handle_type handle, std::size_t off) const noexcept { + return handle + off; + } + + int member; +}; + +static_assert(check_accessor_policy_requirements>()); +static_assert(std::is_trivially_copyable_v>); +static_assert(std::is_trivially_default_constructible_v>); + namespace detail { template constexpr void check_members_with_mixed_extents(Fn&& fn) { diff --git a/tests/std/test.lst b/tests/std/test.lst index f4b02966ba2..079235d8159 100644 --- a/tests/std/test.lst +++ b/tests/std/test.lst @@ -189,6 +189,7 @@ tests\GH_001103_countl_zero_correctness tests\GH_001105_custom_streambuf_throws tests\GH_001123_random_cast_out_of_range tests\GH_001277_num_get_bad_grouping +tests\GH_001394_msvc_no_unique_address_23 tests\GH_001411_core_headers tests\GH_001530_binomial_accuracy tests\GH_001541_case_sensitive_boolalpha @@ -282,6 +283,7 @@ tests\P0009R18_mdspan_layout_stride tests\P0009R18_mdspan_layout_stride_death tests\P0009R18_mdspan_mdspan tests\P0009R18_mdspan_mdspan_death +tests\P0009R18_mdspan_msabi tests\P0019R8_atomic_ref tests\P0024R2_parallel_algorithms_adjacent_difference tests\P0024R2_parallel_algorithms_adjacent_find diff --git a/tests/std/tests/GH_001394_msvc_no_unique_address_23/env.lst b/tests/std/tests/GH_001394_msvc_no_unique_address_23/env.lst new file mode 100644 index 00000000000..642f530ffad --- /dev/null +++ b/tests/std/tests/GH_001394_msvc_no_unique_address_23/env.lst @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +RUNALL_INCLUDE ..\usual_latest_matrix.lst diff --git a/tests/std/tests/GH_001394_msvc_no_unique_address_23/test.compile.pass.cpp b/tests/std/tests/GH_001394_msvc_no_unique_address_23/test.compile.pass.cpp new file mode 100644 index 00000000000..a9bf3b07850 --- /dev/null +++ b/tests/std/tests/GH_001394_msvc_no_unique_address_23/test.compile.pass.cpp @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Tests MSVC STL specific behavior on ABI. + +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +static_assert(sizeof(ranges::in_value_result) == sizeof(int)); +static_assert(sizeof(ranges::in_value_result) == sizeof(int)); +#ifndef __EDG__ // TRANSITION, DevCom-10747012 +static_assert(is_empty_v>); +#endif // ^^^ no workaround ^^^ + +static_assert(sizeof(ranges::out_value_result) == sizeof(int)); +static_assert(sizeof(ranges::out_value_result) == sizeof(int)); +#ifndef __EDG__ // TRANSITION, DevCom-10747012 +static_assert(is_empty_v>); +#endif // ^^^ no workaround ^^^ + +struct stateless_input_it { + using value_type = int; + using difference_type = ptrdiff_t; + + int operator*() const noexcept { + return 42; + } + + stateless_input_it& operator++() noexcept { + return *this; + } + + stateless_input_it operator++(int) noexcept { + return {}; + } +}; + +struct stateless_forward_it { + using value_type = int; + using difference_type = ptrdiff_t; + + int operator*() const noexcept { + return 42; + } + + stateless_forward_it& operator++() noexcept { + return *this; + } + + stateless_forward_it operator++(int) noexcept { + return {}; + } + + friend bool operator==(stateless_forward_it, stateless_forward_it) = default; +}; + +static_assert(input_iterator); +static_assert(forward_iterator); + +using test_range = ranges::subrange; +using test_outer_range = ranges::subrange; +using stateless_input_range = ranges::subrange; +using stateless_fwd_range = ranges::subrange; +using stateless_fwd_common_range = ranges::subrange; + +#ifndef __EDG__ // TRANSITION, DevCom-10747012 +static_assert(is_empty_v); +#endif // ^^^ no workaround ^^^ + +static_assert(sizeof((stateless_input_range{} | views::enumerate).begin()) == sizeof(ptrdiff_t)); +#ifndef __EDG__ // TRANSITION, DevCom-10747012 +static_assert(is_empty_v); +#endif // ^^^ no workaround ^^^ + +static_assert(sizeof((stateless_fwd_range{} | views::chunk(42)).begin()) == sizeof(ptrdiff_t) * 2); +#ifndef __EDG__ // TRANSITION, DevCom-10747012 +static_assert(sizeof((stateless_fwd_common_range{} | views::chunk(42)).begin()) == sizeof(ptrdiff_t) * 2); +#endif // ^^^ no workaround ^^^ + +static_assert(sizeof((stateless_fwd_range{} | views::chunk_by(ranges::less{})).begin()) == sizeof(void*) * 2); +static_assert(sizeof((stateless_fwd_common_range{} | views::chunk_by(ranges::less{})).begin()) == sizeof(void*) * 2); + +#ifndef __EDG__ // TRANSITION, DevCom-10747012 +static_assert(is_empty_v); +#endif // ^^^ no workaround ^^^ + +static_assert(sizeof(ranges::elements_of>) == sizeof(int[1])); +static_assert(sizeof(ranges::elements_of>) == sizeof(int (*)[1])); +static_assert(sizeof(ranges::elements_of, pmr::polymorphic_allocator>) + == sizeof(pmr::polymorphic_allocator)); +static_assert(sizeof(ranges::elements_of, allocator>) == 1); + +#ifndef __EDG__ // TRANSITION, DevCom-10747012 +static_assert(is_empty_v, allocator>>); +#endif // ^^^ no workaround ^^^ diff --git a/tests/std/tests/P0009R18_mdspan_extents/test.cpp b/tests/std/tests/P0009R18_mdspan_extents/test.cpp index 2dedead9cc9..90163bfe391 100644 --- a/tests/std/tests/P0009R18_mdspan_extents/test.cpp +++ b/tests/std/tests/P0009R18_mdspan_extents/test.cpp @@ -545,12 +545,6 @@ static_assert(all_extents_dynamic, 7>); static_assert(all_extents_dynamic, 8>); static_assert(all_extents_dynamic, 9>); -// When 'E::rank_dynamic()' is equal to 0 then 'is_empty_v' should be true (MSVC STL specific behavior) -static_assert(!is_empty_v>); -static_assert(!is_empty_v>); -static_assert(is_empty_v>); -static_assert(is_empty_v>); - int main() { static_assert(test()); test(); diff --git a/tests/std/tests/P0009R18_mdspan_layout_left/test.cpp b/tests/std/tests/P0009R18_mdspan_layout_left/test.cpp index b26ca701f86..90d00be88b7 100644 --- a/tests/std/tests/P0009R18_mdspan_layout_left/test.cpp +++ b/tests/std/tests/P0009R18_mdspan_layout_left/test.cpp @@ -486,12 +486,6 @@ constexpr void check_correctness() { } } -// When 'M::extents_type::rank_dynamic()' is equal to 0 then 'is_empty_v' should be true (MSVC STL specific behavior) -static_assert(!is_empty_v>>); -static_assert(!is_empty_v>>); -static_assert(is_empty_v>>); -static_assert(is_empty_v>>); - constexpr bool test() { check_members_with_various_extents([](const E& e) { check_members(e, make_index_sequence{}); }); if (!is_constant_evaluated()) { // too heavy for compile time diff --git a/tests/std/tests/P0009R18_mdspan_layout_right/test.cpp b/tests/std/tests/P0009R18_mdspan_layout_right/test.cpp index 89899ab408c..fa3679ff03f 100644 --- a/tests/std/tests/P0009R18_mdspan_layout_right/test.cpp +++ b/tests/std/tests/P0009R18_mdspan_layout_right/test.cpp @@ -517,12 +517,6 @@ constexpr void check_correctness() { } } -// When 'M::extents_type::rank_dynamic()' is equal to 0 then 'is_empty_v' should be true (MSVC STL specific behavior) -static_assert(!is_empty_v>>); -static_assert(!is_empty_v>>); -static_assert(is_empty_v>>); -static_assert(is_empty_v>>); - constexpr bool test() { check_members_with_various_extents([](const E& e) { check_members(e, make_index_sequence{}); }); if (!is_constant_evaluated()) { // too heavy for compile time diff --git a/tests/std/tests/P0009R18_mdspan_layout_stride/test.cpp b/tests/std/tests/P0009R18_mdspan_layout_stride/test.cpp index f2dadd09de6..dc6daa67e0a 100644 --- a/tests/std/tests/P0009R18_mdspan_layout_stride/test.cpp +++ b/tests/std/tests/P0009R18_mdspan_layout_stride/test.cpp @@ -821,12 +821,6 @@ constexpr void check_correctness() { } } -// When 'M::extents_type::rank()' is equal to 0 then 'is_empty_v' should be true (MSVC STL specific behavior) -static_assert(!is_empty_v>>); -static_assert(!is_empty_v>>); -static_assert(!is_empty_v>>); -static_assert(is_empty_v>>); - constexpr bool test() { // Check signed integers check_members(extents{5}, array{1}); diff --git a/tests/std/tests/P0009R18_mdspan_mdspan/test.cpp b/tests/std/tests/P0009R18_mdspan_mdspan/test.cpp index 330d3cadc34..a36411d53e8 100644 --- a/tests/std/tests/P0009R18_mdspan_mdspan/test.cpp +++ b/tests/std/tests/P0009R18_mdspan_mdspan/test.cpp @@ -318,28 +318,6 @@ struct AccessorWithCustomOffsetPolicy { static_assert(check_accessor_policy_requirements>()); -template -struct TrivialAccessor { - using offset_policy = TrivialAccessor; - using element_type = ElementType; - using reference = ElementType&; - using data_handle_type = ElementType*; - - constexpr reference access(data_handle_type handle, size_t off) const noexcept { - return handle[off]; - } - - constexpr data_handle_type offset(data_handle_type handle, size_t off) const noexcept { - return handle + off; - } - - int member; -}; - -static_assert(check_accessor_policy_requirements>()); -static_assert(is_trivially_copyable_v>); -static_assert(is_trivially_default_constructible_v>); - template class AccessorTemplate> constexpr void check_modeled_concepts_and_member_types() { using Accessor = AccessorTemplate; @@ -1360,25 +1338,6 @@ constexpr void check_deduction_guides() { } } -// When -// * 'Mds::accessor_type' is a specialization of 'default_accessor', and -// * 'Mds::layout_type' is -// * 'layout_left' or 'layout_right' and 'Mds::extents_type::rank_dynamic() == 0', or -// * 'layout_stride' and 'Mds::extents_type::rank() == 0' -// then 'sizeof(Mds) == sizeof(void*)' (MSVC STL specific behavior). -static_assert(sizeof(mdspan, layout_left>) == sizeof(void*)); -static_assert(sizeof(mdspan, layout_left>) > sizeof(void*)); -static_assert(sizeof(mdspan, layout_left, TrivialAccessor>) > sizeof(void*)); - -static_assert(sizeof(mdspan, layout_right>) == sizeof(void*)); -static_assert(sizeof(mdspan, layout_right>) > sizeof(void*)); -static_assert(sizeof(mdspan, layout_right, TrivialAccessor>) > sizeof(void*)); - -static_assert(sizeof(mdspan, layout_stride>) == sizeof(void*)); -static_assert(sizeof(mdspan, layout_stride>) > sizeof(void*)); -static_assert(sizeof(mdspan, layout_stride>) > sizeof(void*)); -static_assert(sizeof(mdspan, layout_stride, TrivialAccessor>) > sizeof(void*)); - constexpr bool test() { check_modeled_concepts_and_member_types, layout_left, default_accessor>(); check_modeled_concepts_and_member_types, layout_right, default_accessor>(); diff --git a/tests/std/tests/P0009R18_mdspan_msabi/env.lst b/tests/std/tests/P0009R18_mdspan_msabi/env.lst new file mode 100644 index 00000000000..642f530ffad --- /dev/null +++ b/tests/std/tests/P0009R18_mdspan_msabi/env.lst @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +RUNALL_INCLUDE ..\usual_latest_matrix.lst diff --git a/tests/std/tests/P0009R18_mdspan_msabi/test.compile.pass.cpp b/tests/std/tests/P0009R18_mdspan_msabi/test.compile.pass.cpp new file mode 100644 index 00000000000..0d85c79d2be --- /dev/null +++ b/tests/std/tests/P0009R18_mdspan_msabi/test.compile.pass.cpp @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Tests MSVC STL specific behavior on ABI. + +#include +#include + +#include + +using namespace std; + +// When 'E::rank_dynamic()' is equal to 0 then 'is_empty_v' should be true +static_assert(!is_empty_v>); +static_assert(!is_empty_v>); +static_assert(is_empty_v>); +static_assert(is_empty_v>); + +// When 'M::extents_type::rank_dynamic()' is equal to 0 then 'is_empty_v' should be true +static_assert(!is_empty_v>>); +static_assert(!is_empty_v>>); +static_assert(is_empty_v>>); +static_assert(is_empty_v>>); + +// When 'M::extents_type::rank_dynamic()' is equal to 0 then 'is_empty_v' should be true +static_assert(!is_empty_v>>); +static_assert(!is_empty_v>>); +static_assert(is_empty_v>>); +static_assert(is_empty_v>>); + +// When 'M::extents_type::rank()' is equal to 0 then 'is_empty_v' should be true +static_assert(!is_empty_v>>); +static_assert(!is_empty_v>>); +static_assert(!is_empty_v>>); +static_assert(is_empty_v>>); + +// When +// * 'Mds::accessor_type' is a specialization of 'default_accessor', and +// * 'Mds::layout_type' is +// * 'layout_left' or 'layout_right' and 'Mds::extents_type::rank_dynamic() == 0', or +// * 'layout_stride' and 'Mds::extents_type::rank() == 0' +// then 'sizeof(Mds) == sizeof(void*)'. +static_assert(sizeof(mdspan, layout_left>) == sizeof(void*)); +static_assert(sizeof(mdspan, layout_left>) > sizeof(void*)); +static_assert(sizeof(mdspan, layout_left, TrivialAccessor>) > sizeof(void*)); + +static_assert(sizeof(mdspan, layout_right>) == sizeof(void*)); +static_assert(sizeof(mdspan, layout_right>) > sizeof(void*)); +static_assert(sizeof(mdspan, layout_right, TrivialAccessor>) > sizeof(void*)); + +static_assert(sizeof(mdspan, layout_stride>) == sizeof(void*)); +static_assert(sizeof(mdspan, layout_stride>) > sizeof(void*)); +static_assert(sizeof(mdspan, layout_stride>) > sizeof(void*)); +static_assert(sizeof(mdspan, layout_stride, TrivialAccessor>) > sizeof(void*)); From 162a42bfa1ed4a9f34ea72ddfd2d2434e8ec4457 Mon Sep 17 00:00:00 2001 From: Gabriel Augusto <95982237+gabrielaugz@users.noreply.github.com> Date: Tue, 25 Mar 2025 00:26:03 -0300 Subject: [PATCH 12/14] Update README.md to reference N5008 instead of N5001 (#5344) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1b83c5cadf0..aaecf28e1bc 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ issue. The [bug tag][] and [enhancement tag][] are being populated. # Goals -We're implementing the latest C++ Working Draft, currently [N5001][], which will eventually become the next C++ +We're implementing the latest C++ Working Draft, currently [N5008][], which will eventually become the next C++ International Standard. The terms Working Draft (WD) and Working Paper (WP) are interchangeable; we often informally refer to these drafts as "the Standard" while being aware of the difference. (There are other relevant Standards; for example, supporting `/std:c++14` and `/std:c++17` involves understanding how the C++14 and C++17 @@ -568,7 +568,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception [LWG issues]: https://cplusplus.github.io/LWG/lwg-toc.html [LWG tag]: https://github.com/microsoft/STL/issues?q=is%3Aopen+is%3Aissue+label%3ALWG [Microsoft Open Source Code of Conduct]: https://opensource.microsoft.com/codeofconduct/ -[N5001]: https://wg21.link/N5001 +[N5008]: https://wg21.link/N5008 [NOTICE.txt]: NOTICE.txt [Ninja]: https://ninja-build.org [STL-CI-badge]: https://dev.azure.com/vclibs/STL/_apis/build/status%2FSTL-CI?branchName=main "STL-CI" From f2a2933dd65d9e8d3fa698a97b6074f7ef00e1fd Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 24 Mar 2025 20:27:13 -0700 Subject: [PATCH 13/14] Update llvm-project (#5349) --- llvm-project | 2 +- stl/inc/vector | 10 +- tests/libcxx/expected_results.txt | 183 ++++++++++++++---- .../test.cpp | 2 + tests/utils/stl/test/tests.py | 6 + 5 files changed, 164 insertions(+), 39 deletions(-) diff --git a/llvm-project b/llvm-project index 63d3bd6d0ca..14006c8b5be 160000 --- a/llvm-project +++ b/llvm-project @@ -1 +1 @@ -Subproject commit 63d3bd6d0caf8185aba49540fe2f67512fdf3a98 +Subproject commit 14006c8b5be12d24d8af2ff4bf4d152b1a2306f5 diff --git a/stl/inc/vector b/stl/inc/vector index a4dcee2cfbe..8d3f31bb0b4 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -3731,22 +3731,22 @@ _NODISCARD _CONSTEXPR20 _VbIt _Find_vbool(_VbIt _First, const _VbIt _Last, const const auto _SourceMask = _FirstSourceMask & _LastSourceMask; const auto _SelectVal = (_Val ? *_VbFirst : ~*_VbFirst) & _SourceMask; const auto _Count = _Countr_zero(_SelectVal); - return _Count == _VBITS ? _Last : _First + static_cast(_Count - _First._Myoff); + return _Count == _VBITS ? _Last : _First + static_cast<_Iter_diff_t<_VbIt>>(_Count - _First._Myoff); } const auto _FirstVal = (_Val ? *_VbFirst : ~*_VbFirst) & _FirstSourceMask; const auto _FirstCount = _Countr_zero(_FirstVal); if (_FirstCount != _VBITS) { - return _First + static_cast(_FirstCount - _First._Myoff); + return _First + static_cast<_Iter_diff_t<_VbIt>>(_FirstCount - _First._Myoff); } ++_VbFirst; - _Iter_diff_t<_VbIt> _TotalCount = static_cast(_VBITS - _First._Myoff); + auto _TotalCount = static_cast<_Iter_diff_t<_VbIt>>(_VBITS - _First._Myoff); for (; _VbFirst != _VbLast; ++_VbFirst, _TotalCount += _VBITS) { const auto _SelectVal = _Val ? *_VbFirst : ~*_VbFirst; const auto _Count = _Countr_zero(_SelectVal); if (_Count != _VBITS) { - return _First + (_TotalCount + _Count); + return _First + static_cast<_Iter_diff_t<_VbIt>>(_TotalCount + _Count); } } @@ -3755,7 +3755,7 @@ _NODISCARD _CONSTEXPR20 _VbIt _Find_vbool(_VbIt _First, const _VbIt _Last, const const auto _LastVal = (_Val ? *_VbFirst : ~*_VbFirst) & _LastSourceMask; const auto _Count = _Countr_zero(_LastVal); if (_Count != _VBITS) { - return _First + (_TotalCount + _Count); + return _First + static_cast<_Iter_diff_t<_VbIt>>(_TotalCount + _Count); } } diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index bf8daabd5c6..3285f4d32ae 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -7,12 +7,18 @@ std/time/time.syn/formatter.duration.pass.cpp:0 FAIL std/time/time.syn/formatter.duration.pass.cpp:1 FAIL std/time/time.syn/formatter.file_time.pass.cpp:0 FAIL std/time/time.syn/formatter.file_time.pass.cpp:1 FAIL +std/time/time.syn/formatter.gps_time.pass.cpp:0 FAIL +std/time/time.syn/formatter.gps_time.pass.cpp:1 FAIL std/time/time.syn/formatter.hh_mm_ss.pass.cpp:0 FAIL std/time/time.syn/formatter.hh_mm_ss.pass.cpp:1 FAIL std/time/time.syn/formatter.local_time.pass.cpp:0 FAIL std/time/time.syn/formatter.local_time.pass.cpp:1 FAIL std/time/time.syn/formatter.sys_time.pass.cpp:0 FAIL std/time/time.syn/formatter.sys_time.pass.cpp:1 FAIL +std/time/time.syn/formatter.tai_time.pass.cpp:0 FAIL +std/time/time.syn/formatter.tai_time.pass.cpp:1 FAIL +std/time/time.syn/formatter.utc_time.pass.cpp:0 FAIL +std/time/time.syn/formatter.utc_time.pass.cpp:1 FAIL std/time/time.syn/formatter.year.pass.cpp:0 FAIL std/time/time.syn/formatter.year.pass.cpp:1 FAIL std/time/time.syn/formatter.year_month.pass.cpp:0 FAIL @@ -36,14 +42,13 @@ std/numerics/rand/rand.dist/rand.dist.uni/rand.dist.uni.real/param_ctor.pass.cpp # LLVM-113609: [libc++][test] Non-rebindable test_alloc in string.capacity/deallocate_size.pass.cpp std/strings/basic.string/string.capacity/deallocate_size.pass.cpp FAIL -# LLVM-116054: [libc++] vprint_[non]unicode(ostream &, string_view, Args&&...) should not pad the output -std/input.output/iostream.format/output.streams/ostream.formatted/ostream.formatted.print/print.pass.cpp FAIL -std/input.output/iostream.format/output.streams/ostream.formatted/ostream.formatted.print/vprint_nonunicode.pass.cpp FAIL -std/input.output/iostream.format/output.streams/ostream.formatted/ostream.formatted.print/vprint_unicode.pass.cpp FAIL - # LLVM-122638: [libc++][test] re.regex.construct/bad_backref.pass.cpp assumes non-standard extension to extended regular expressions std/re/re.regex/re.regex.construct/bad_backref.pass.cpp FAIL +# LLVM-132532: [libc++][test] Small fixes for time tests +std/time/time.clock/time.clock.gps/types.compile.pass.cpp:2 FAIL +std/time/time.clock/time.clock.tai/types.compile.pass.cpp:2 FAIL + # Non-Standard regex behavior. # "It seems likely that the test is still non-conforming due to how libc++ handles the 'w' character class." std/re/re.traits/lookup_classname.pass.cpp FAIL @@ -226,6 +231,7 @@ std/depr/depr.c.headers/uchar_h.compile.pass.cpp FAIL std/strings/c.strings/cuchar.compile.pass.cpp FAIL # P0429R9 +std/containers/container.adaptors/flat.map.syn/sorted_equivalent.pass.cpp FAIL std/containers/container.adaptors/flat.map.syn/sorted_unique.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.access/at_transparent.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.access/at.pass.cpp FAIL @@ -244,6 +250,7 @@ std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.addressof.c std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.cons/copy.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.cons/deduct_pmr.pass.cpp FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/deduct.compile.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.cons/default_noexcept.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.cons/default.pass.cpp FAIL @@ -311,13 +318,107 @@ std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound.pass. std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp FAIL std/containers/container.adaptors/flat.map/op_compare.pass.cpp FAIL std/containers/container.adaptors/flat.map/types.compile.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/empty.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/max_size.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/size.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/alloc.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/assign_initializer_list.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/compare.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/containers.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_alloc.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.addressof.compile.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct_pmr.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.compile.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/default_noexcept.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/default.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/dtor_noexcept.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/initializer_list.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/iter_iter.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_alloc.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_clears.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_noexcept.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_exceptions.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_noexcept.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/pmr.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/range.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_container.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_initializer_list.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_iter_iter.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.erasure/erase_if_exceptions.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.erasure/erase_if.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator_comparison.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator_concept_conformance.compile.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/iterator.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/range_concept_conformance.compile.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.iterators/reverse_iterator.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/clear.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/emplace_hint.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/emplace.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_iter_iter.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_iter.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key_transparent.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/extract.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_cv.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_initializer_list.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_cv.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_rv.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_rv.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_initializer_list.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_iter_iter.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_transparent.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/replace.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_free.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_member.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.observers/comp.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.observers/keys_values.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains_transparent.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/contains.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count_transparent.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/count.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range_transparent.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/equal_range.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find_transparent.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/find.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound_transparent.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/lower_bound.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound_transparent.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/incomplete_type.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/op_compare.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/types.compile.pass.cpp FAIL +std/language.support/support.limits/support.limits.general/flat_map.version.compile.pass.cpp FAIL +std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp FAIL +std/utilities/format/format.range/format.range.fmtmap/format.functions.format.pass.cpp FAIL +std/utilities/format/format.range/format.range.fmtmap/format.functions.vformat.pass.cpp FAIL # P0533R9 constexpr For And std/language.support/support.limits/support.limits.general/cmath.version.compile.pass.cpp FAIL std/language.support/support.limits/support.limits.general/cstdlib.version.compile.pass.cpp FAIL +# P0543R3 Saturation Arithmetic +std/numerics/numeric.ops/numeric.ops.sat/add_sat.compile.pass.cpp FAIL +std/numerics/numeric.ops/numeric.ops.sat/add_sat.pass.cpp FAIL +std/numerics/numeric.ops/numeric.ops.sat/div_sat.compile.pass.cpp FAIL +std/numerics/numeric.ops/numeric.ops.sat/div_sat.pass.cpp FAIL +std/numerics/numeric.ops/numeric.ops.sat/mul_sat.compile.pass.cpp FAIL +std/numerics/numeric.ops/numeric.ops.sat/mul_sat.pass.cpp FAIL +std/numerics/numeric.ops/numeric.ops.sat/saturate_cast.compile.pass.cpp FAIL +std/numerics/numeric.ops/numeric.ops.sat/saturate_cast.pass.cpp FAIL +std/numerics/numeric.ops/numeric.ops.sat/sub_sat.compile.pass.cpp FAIL +std/numerics/numeric.ops/numeric.ops.sat/sub_sat.pass.cpp FAIL + # P2255R2 "Type Traits To Detect References Binding To Temporaries" std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp FAIL +std/utilities/meta/meta.unary/meta.unary.prop/reference_constructs_from_temporary.pass.cpp FAIL +std/utilities/meta/meta.unary/meta.unary.prop/reference_converts_from_temporary.pass.cpp FAIL # P2674R1 is_implicit_lifetime std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp FAIL @@ -444,6 +545,10 @@ std/strings/basic.string/string.modifiers/string_replace/iter_iter_string.pass.c std/strings/basic.string/string.modifiers/string_replace/iter_iter_string.pass.cpp:1 FAIL std/strings/basic.string/string.modifiers/string_replace/iter_iter_string_view.pass.cpp:0 FAIL std/strings/basic.string/string.modifiers/string_replace/iter_iter_string_view.pass.cpp:1 FAIL +std/strings/strings.erasure/erase_if.pass.cpp:0 FAIL +std/strings/strings.erasure/erase_if.pass.cpp:1 FAIL +std/strings/strings.erasure/erase.pass.cpp:0 FAIL +std/strings/strings.erasure/erase.pass.cpp:1 FAIL # VSO-2338880 constexpr error in vector::iterator's _Compat() check when using views::transform std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp:0 FAIL @@ -551,6 +656,13 @@ std/input.output/string.streams/stringbuf/stringbuf.members/view.pass.cpp FAIL std/input.output/syncstream/syncbuf/syncstream.syncbuf.cons/dtor.pass.cpp FAIL std/input.output/syncstream/syncbuf/syncstream.syncbuf.members/emit.pass.cpp FAIL +# GH-5345: : _Copy_vbool() mishandles vectors of size 32 and 64, revealed by constexpr Clang +std/algorithms/alg.modifying.operations/alg.copy/copy_n.pass.cpp:2 FAIL +std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp:2 FAIL +std/algorithms/alg.modifying.operations/alg.move/move.pass.cpp:2 FAIL +std/algorithms/alg.nonmodifying/alg.equal/equal.pass.cpp FAIL +std/algorithms/alg.nonmodifying/alg.equal/ranges.equal.pass.cpp FAIL + # *** VCRUNTIME BUGS *** @@ -605,6 +717,13 @@ std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.lockin std/thread/thread.threads/thread.thread.class/thread.thread.destr/dtor.pass.cpp SKIPPED std/thread/thread.threads/thread.thread.this/sleep_until.pass.cpp SKIPPED +# "This is technically flaky" comments indicate tests with timing assumptions. +# These tests have failed in practice, see VSO-2321213 and VSO-2416940. +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_for.pass.cpp SKIPPED +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_for.pass.cpp SKIPPED +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_shared_until.pass.cpp SKIPPED +std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requirements/thread.sharedtimedmutex.class/try_lock_until.pass.cpp SKIPPED + # Not analyzed, likely bogus tests. Various assertions, probably POSIX assumptions. std/diagnostics/syserr/syserr.syserr/syserr.syserr.members/ctor_error_code_const_char_pointer.pass.cpp FAIL std/diagnostics/syserr/syserr.syserr/syserr.syserr.members/ctor_error_code_string.pass.cpp FAIL @@ -693,6 +812,10 @@ std/containers/sequences/vector/vector.capacity/resize_size_value_exceptions.pas # error: incomplete type 'std::char_traits' named in nested name specifier std/iterators/iterator.requirements/iterator.assoc.types/readable.traits/indirectly_readable_traits.compile.pass.cpp FAIL +# warning C5030: attribute [[clang::trivial_abi]] is not recognized +std/containers/sequences/vector/trivial_relocation.pass.cpp:0 FAIL +std/containers/sequences/vector/trivial_relocation.pass.cpp:1 FAIL + # *** LIKELY STL BUGS *** # Not analyzed, likely STL bugs. Various assertions. @@ -868,22 +991,16 @@ std/numerics/rand/rand.dist/rand.dist.pois/rand.dist.pois.poisson/eval.pass.cpp # I don't know which is right. std/language.support/support.limits/limits/numeric.limits.members/tinyness_before.pass.cpp SKIPPED -# Not analyzed. the test fails on x64 but passes on x86 -# warning C4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data -# warning C4244: 'initializing': conversion from 'unsigned __int64' to 'int', possible loss of data +# Not analyzed. x64-specific truncation warnings. +std/algorithms/alg.modifying.operations/alg.fill/fill_n.pass.cpp:0 SKIPPED +std/algorithms/alg.modifying.operations/alg.fill/fill_n.pass.cpp:1 SKIPPED +std/algorithms/alg.modifying.operations/alg.random.shuffle/ranges_shuffle.pass.cpp:0 SKIPPED +std/algorithms/alg.modifying.operations/alg.random.shuffle/ranges_shuffle.pass.cpp:1 SKIPPED std/containers/associative/map/map.access/iterator.pass.cpp:0 SKIPPED std/containers/associative/map/map.access/iterator.pass.cpp:1 SKIPPED - -# Not analyzed. the test fails on x64 but passes on x86 -# warning C4244: 'return': conversion from '__int64' to 'CustomIt::difference_type', possible loss of data std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.-/sentinel.pass.cpp:0 SKIPPED std/iterators/predef.iterators/move.iterators/move.iter.ops/move.iter.op.-/sentinel.pass.cpp:1 SKIPPED -# Not analyzed. the test fails on x64 but passes on x86 -# warning C4267: 'initializing': conversion from 'size_t' to 'int', possible loss of data -std/algorithms/alg.modifying.operations/alg.random.shuffle/ranges_shuffle.pass.cpp:0 SKIPPED -std/algorithms/alg.modifying.operations/alg.random.shuffle/ranges_shuffle.pass.cpp:1 SKIPPED - # Not analyzed. the test fails on x86 but passes on x64 # warning C4389: '==': signed/unsigned mismatch std/algorithms/alg.modifying.operations/alg.unique/ranges_unique.pass.cpp:0 SKIPPED @@ -949,6 +1066,8 @@ std/thread/thread.mutex/thread.mutex.requirements/thread.sharedtimedmutex.requir std/utilities/utility/mem.res/mem.poly.allocator.class/mem.poly.allocator.mem/construct_piecewise_pair_evil.pass.cpp FAIL # Not analyzed, failing due to constexpr step limits. +std/algorithms/alg.modifying.operations/alg.copy/ranges.copy_backward.pass.cpp:2 FAIL +std/algorithms/alg.modifying.operations/alg.rotate/rotate.pass.cpp FAIL std/algorithms/alg.nonmodifying/alg.contains/ranges.contains_subrange.pass.cpp:2 FAIL std/algorithms/alg.nonmodifying/alg.count/count.pass.cpp FAIL std/algorithms/alg.nonmodifying/alg.count/ranges.count.pass.cpp FAIL @@ -1041,9 +1160,12 @@ std/localization/codecvt_unicode.pass.cpp FAIL # Not analyzed. Various runtime behavior differences. std/time/time.syn/formatter.duration.pass.cpp:2 FAIL std/time/time.syn/formatter.file_time.pass.cpp:2 FAIL +std/time/time.syn/formatter.gps_time.pass.cpp:2 FAIL std/time/time.syn/formatter.hh_mm_ss.pass.cpp:2 FAIL std/time/time.syn/formatter.local_time.pass.cpp:2 FAIL std/time/time.syn/formatter.sys_time.pass.cpp:2 FAIL +std/time/time.syn/formatter.tai_time.pass.cpp:2 FAIL +std/time/time.syn/formatter.utc_time.pass.cpp:2 FAIL std/time/time.syn/formatter.zoned_time.pass.cpp:2 FAIL # Not analyzed. constexpr evaluation fails with note: failure was caused by out of range index MEOW; allowed range is 0 <= index < 2 @@ -1072,6 +1194,10 @@ std/strings/char.traits/char.traits.specializations/char.traits.specializations. std/containers/views/mdspan/mdspan/index_operator.pass.cpp:0 FAIL std/containers/views/mdspan/mdspan/index_operator.pass.cpp:1 FAIL +# Not analyzed. Likely MSVC constexpr bug, "note: a non-constant (sub-)expression was encountered" in swap() +std/utilities/utility/utility.swap/swap_array.pass.cpp:0 FAIL +std/utilities/utility/utility.swap/swap_array.pass.cpp:1 FAIL + # Not analyzed. SKIPPED because this is x86-specific, fatal error C1060: compiler is out of heap space std/algorithms/alg.modifying.operations/alg.transform/ranges.transform.binary.iterator.pass.cpp:0 SKIPPED std/algorithms/alg.modifying.operations/alg.transform/ranges.transform.binary.iterator.pass.cpp:1 SKIPPED @@ -1241,8 +1367,10 @@ std/atomics/atomics.ref/wait.pass.cpp SKIPPED std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.const/pointer_deleter.pass.cpp FAIL # Not analyzed. Various assertions. +std/time/time.clock/time.clock.gps/gps_time.ostream.pass.cpp FAIL +std/time/time.clock/time.clock.tai/tai_time.ostream.pass.cpp FAIL +std/time/time.clock/time.clock.utc/time.clock.utc.members/to_sys.pass.cpp FAIL std/time/time.zone/time.zone.timezone/time.zone.members/get_info.local_time.pass.cpp FAIL -std/time/time.zone/time.zone.timezone/time.zone.members/to_sys_choose.pass.cpp FAIL std/time/time.zone/time.zone.zonedtime/time.zone.zonedtime.members/get_info.pass.cpp FAIL std/time/time.zone/time.zone.zonedtime/time.zone.zonedtime.nonmembers/ostream.pass.cpp FAIL @@ -1269,6 +1397,9 @@ std/depr/depr.c.headers/stdalign_h.compile.pass.cpp:1 FAIL std/ranges/range.adaptors/range.adaptor.object/range_adaptor_closure.pass.cpp:0 FAIL std/ranges/range.adaptors/range.adaptor.object/range_adaptor_closure.pass.cpp:1 FAIL +# Not analyzed. Emitting easily-suppressed truncation warnings for MSVC, but also compiler errors for MSVC and Clang. +std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp FAIL + # *** XFAILS WHICH PASS *** # These tests contain `// XFAIL: msvc` comments, which accurately describe runtime failures for x86 and x64. @@ -1331,16 +1462,10 @@ std/ranges/range.utility/range.utility.conv/to.pass.cpp:9 SKIPPED std/thread/thread.jthread/assign.move.pass.cpp:9 SKIPPED std/utilities/meta/meta.unary/dependent_return_type.compile.pass.cpp:9 SKIPPED -# This test is marked as `XFAIL: msvc`, but the MSVC-internal test harness doesn't yet parse XFAIL. -std/utilities/function.objects/func.wrap/func.wrap.func/noncopyable_return_type.pass.cpp:9 SKIPPED - -# Similarly, this test is marked as `XFAIL: msvc` -# MSVC doesn't properly support [[no_unique_address]] +# These tests are marked as `XFAIL: msvc`, but the MSVC-internal test harness doesn't yet parse XFAIL. std/algorithms/algorithms.results/no_unique_address.compile.pass.cpp:9 SKIPPED - -# Similarly, this test is marked as `XFAIL: msvc, target={{.+}}-windows-gnu` -# because it calls an `fmemopen` function that doesn't exist on Windows. std/input.output/iostream.format/print.fun/no_file_description.pass.cpp:9 SKIPPED +std/utilities/function.objects/func.wrap/func.wrap.func/noncopyable_return_type.pass.cpp:9 SKIPPED # This test is marked as `REQUIRES: large_tests`. The GitHub test harness doesn't define that feature, so it doesn't # run this test. However, the MSVC-internal test harness doesn't yet parse REQUIRES, and simply runs everything. @@ -1349,14 +1474,6 @@ std/input.output/string.streams/stringstream/stringstream.members/gcount.pass.cp # Similarly, this test is marked as `REQUIRES: 32-bit-pointer`. std/iterators/iterator.container/ssize.LWG3207.compile.pass.cpp:9 SKIPPED -# This test is marked as `REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-vector`. -# Listing these on separate lines would allow magic_comments.txt to recognize it. -std/containers/sequences/vector/vector.modifiers/assert.push_back.invalidation.pass.cpp:9 SKIPPED - -# This test is marked as `REQUIRES: has-unix-headers, libcpp-has-abi-bounded-iterators-in-std-array`. -# Listing these on separate lines would allow magic_comments.txt to recognize it. -std/containers/sequences/array/assert.iterators.pass.cpp:9 SKIPPED - # These tests emit C5321, which warns when the resolution to CWG-1656 affects a u8 string literal. # The conformant behavior is opt-in because it can silently change behavior. # warning C5321: nonstandard extension used: encoding '\x80' as a multi-byte utf-8 character. Use \u instead diff --git a/tests/std/tests/Dev11_1127004_future_has_exceptions_0/test.cpp b/tests/std/tests/Dev11_1127004_future_has_exceptions_0/test.cpp index 08481988c50..7701e9aeef4 100644 --- a/tests/std/tests/Dev11_1127004_future_has_exceptions_0/test.cpp +++ b/tests/std/tests/Dev11_1127004_future_has_exceptions_0/test.cpp @@ -4,5 +4,7 @@ #include int main() { +#ifndef _M_CEE // TRANSITION, VSO-2359637, avoid sporadic failures under /clr std::async([] {}).wait(); +#endif } diff --git a/tests/utils/stl/test/tests.py b/tests/utils/stl/test/tests.py index 658db996127..ef51b790d4a 100644 --- a/tests/utils/stl/test/tests.py +++ b/tests/utils/stl/test/tests.py @@ -275,10 +275,16 @@ def _parseFlags(self, litConfig): foundStd = True if flag[5:] == 'c++latest': self._addCustomFeature('c++23') + self._addCustomFeature('std-at-least-c++23') + self._addCustomFeature('std-at-least-c++20') + self._addCustomFeature('std-at-least-c++17') elif flag[5:] == 'c++20': self._addCustomFeature('c++20') + self._addCustomFeature('std-at-least-c++20') + self._addCustomFeature('std-at-least-c++17') elif flag[5:] == 'c++17': self._addCustomFeature('c++17') + self._addCustomFeature('std-at-least-c++17') elif flag[5:] == 'c++14': self._addCustomFeature('c++14') elif flag[1:11] == 'fsanitize=': From e54056fdd31dcb8db717c354ce82b71299e5a35e Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Mon, 24 Mar 2025 21:08:26 -0700 Subject: [PATCH 14/14] Recategorize libcxx failures from scratch. --- tests/libcxx/expected_results.txt | 93 ++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index 2c36f3bc889..f4e1ef53e3b 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -231,9 +231,98 @@ std/depr/depr.c.headers/uchar_h.compile.pass.cpp FAIL std/strings/c.strings/cuchar.compile.pass.cpp FAIL # P0429R9 -# FIXME! Investigate these failures before merging! -# FIXME! These tests perform narrowing conversions on x64 but not on x86. +# FIXME! static_assert(!CanIndex>); +std/containers/container.adaptors/flat.map/flat.map.access/index_transparent.pass.cpp FAIL + +# FIXME! static_assert(!CanEmplace); +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/emplace.pass.cpp FAIL + +# FIXME! error: no type named 'value_type' in 'std::greater' +std/containers/container.adaptors/flat.map/flat.map.cons/deduct.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct.pass.cpp FAIL + +# FIXME! Assertion failed: mo.empty() +std/containers/container.adaptors/flat.map/flat.map.cons/move_alloc.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_alloc.pass.cpp FAIL + +# FIXME! Assertion failed: m.keys().size() == m.values().size() +std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_clears.pass.cpp FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/move_exceptions.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_clears.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_exceptions.pass.cpp FAIL + +# FIXME! abort() has been called +std/containers/container.adaptors/flat.map/flat.map.cons/move.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move.pass.cpp FAIL + +# FIXME! Assertion failed: false +std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_initializer_list.pass.cpp FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range.pass.cpp FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_initializer_list.pass.cpp FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_iter_iter.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_initializer_list.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_initializer_list.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_iter_iter.pass.cpp FAIL + +# FIXME! warning C4242: 'initializing': conversion from '_Ty' to '_Ty2', possible loss of data +std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/copy_assign.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/deduct_pmr.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/deduct_pmr.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/initializer_list.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/initializer_list.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/iter_iter.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/move_assign.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/move_assign.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/pmr.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/pmr.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/range.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/range.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/sorted_container.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/sorted_container.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/sorted_initializer_list.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/sorted_initializer_list.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/sorted_iter_iter.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.cons/sorted_iter_iter.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/erase_key.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/containers.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/containers.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/copy_assign.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct_pmr.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/deduct_pmr.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/initializer_list.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/initializer_list.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/iter_iter.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/iter_iter.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/pmr.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/pmr.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/range.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/range.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_container.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_container.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_initializer_list.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_initializer_list.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_iter_iter.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/sorted_iter_iter.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/erase_key.pass.cpp:1 FAIL + +# FIXME! error: unused type alias 'C' [-Werror,-Wunused-local-typedef] +std/containers/container.adaptors/flat.map/flat.map.cons/move_assign_noexcept.pass.cpp:2 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.cons/move_assign_noexcept.pass.cpp:2 FAIL + +# FIXME! error: unused variable 'expected' [-Werror,-Wunused-variable] +std/containers/container.adaptors/flat.map/flat.map.erasure/erase_if_exceptions.pass.cpp:2 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.erasure/erase_if_exceptions.pass.cpp:2 FAIL # P0533R9 constexpr For And std/language.support/support.limits/support.limits.general/cmath.version.compile.pass.cpp FAIL