From a92ce66e3c637aacb3c30233ca4065a366d8a243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 23 Jul 2021 17:41:20 +0200 Subject: [PATCH 1/4] Quadruple stack size to be able to run full test suite with sanitizers --- configure | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 050f1836011143..e44254460ff23a 100755 --- a/configure +++ b/configure @@ -9814,7 +9814,7 @@ then # to ensure that tests don't crash # Note: This matches the value of THREAD_STACK_SIZE in # thread_pthread.h - LINKFORSHARED="-Wl,-stack_size,1000000 $LINKFORSHARED" + LINKFORSHARED="-Wl,-stack_size,4000000 $LINKFORSHARED" if test "$enable_framework" then diff --git a/configure.ac b/configure.ac index aaff79f720e3c4..3caff87dbd1b0c 100644 --- a/configure.ac +++ b/configure.ac @@ -2800,7 +2800,7 @@ then # to ensure that tests don't crash # Note: This matches the value of THREAD_STACK_SIZE in # thread_pthread.h - LINKFORSHARED="-Wl,-stack_size,1000000 $LINKFORSHARED" + LINKFORSHARED="-Wl,-stack_size,4000000 $LINKFORSHARED" if test "$enable_framework" then From 188315e0b203f9bdf54d9f5d14995e95af6fcd50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 27 Aug 2021 16:17:47 +0200 Subject: [PATCH 2/4] Only quadruple stack size if --with-undefined-behavior-sanitizer was passed --- configure | 143 ++++++++++++++++++++++++++++---------------------- configure.ac | 89 +++++++++++++++++-------------- pyconfig.h.in | 4 ++ 3 files changed, 134 insertions(+), 102 deletions(-) diff --git a/configure b/configure index e44254460ff23a..baa90dc98dd510 100755 --- a/configure +++ b/configure @@ -827,11 +827,11 @@ with_trace_refs with_assertions enable_optimizations with_lto -with_hash_algorithm -with_tzpath with_address_sanitizer with_memory_sanitizer with_undefined_behavior_sanitizer +with_hash_algorithm +with_tzpath with_libs with_system_expat with_system_ffi @@ -1548,12 +1548,6 @@ Optional Packages: --with-lto=[full|thin|no|yes] enable Link-Time-Optimization in any build (default is no) - --with-hash-algorithm=[fnv|siphash24] - select hash algorithm for use in Python/pyhash.c - (default is SipHash24) - --with-tzpath= - Select the default time zone search path for zoneinfo.TZPATH - --with-address-sanitizer enable AddressSanitizer memory error detector, 'asan' (default is no) @@ -1562,6 +1556,12 @@ Optional Packages: --with-undefined-behavior-sanitizer enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no) + --with-hash-algorithm=[fnv|siphash24] + select hash algorithm for use in Python/pyhash.c + (default is SipHash24) + --with-tzpath= + Select the default time zone search path for zoneinfo.TZPATH + --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) @@ -9602,6 +9602,65 @@ $as_echo "no" >&6; } ;; esac +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-address-sanitizer" >&5 +$as_echo_n "checking for --with-address-sanitizer... " >&6; } + +# Check whether --with-address_sanitizer was given. +if test "${with_address_sanitizer+set}" = set; then : + withval=$with_address_sanitizer; +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +$as_echo "$withval" >&6; } +BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="-fsanitize=address $LDFLAGS" +# ASan works by controlling memory allocation, our own malloc interferes. +with_pymalloc="no" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5 +$as_echo_n "checking for --with-memory-sanitizer... " >&6; } + +# Check whether --with-memory_sanitizer was given. +if test "${with_memory_sanitizer+set}" = set; then : + withval=$with_memory_sanitizer; +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +$as_echo "$withval" >&6; } +BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" +# MSan works by controlling memory allocation, our own malloc interferes. +with_pymalloc="no" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-undefined-behavior-sanitizer" >&5 +$as_echo_n "checking for --with-undefined-behavior-sanitizer... " >&6; } + +# Check whether --with-undefined_behavior_sanitizer was given. +if test "${with_undefined_behavior_sanitizer+set}" = set; then : + withval=$with_undefined_behavior_sanitizer; +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +$as_echo "$withval" >&6; } +BASECFLAGS="-fsanitize=undefined $BASECFLAGS" +LDFLAGS="-fsanitize=undefined $LDFLAGS" +with_ubsan="yes" + +else + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +with_ubsan="no" + +fi + + # Set info about shared libraries. @@ -9814,7 +9873,18 @@ then # to ensure that tests don't crash # Note: This matches the value of THREAD_STACK_SIZE in # thread_pthread.h - LINKFORSHARED="-Wl,-stack_size,4000000 $LINKFORSHARED" + stack_size="1000000" + if test "$with_ubsan" == "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" + +cat >>confdefs.h <<_ACEOF +#define THREAD_STACK_SIZE 0x$stack_size +_ACEOF + + fi + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" if test "$enable_framework" then @@ -10410,61 +10480,6 @@ fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-address-sanitizer" >&5 -$as_echo_n "checking for --with-address-sanitizer... " >&6; } - -# Check whether --with-address_sanitizer was given. -if test "${with_address_sanitizer+set}" = set; then : - withval=$with_address_sanitizer; -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 -$as_echo "$withval" >&6; } -BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" -LDFLAGS="-fsanitize=address $LDFLAGS" -# ASan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5 -$as_echo_n "checking for --with-memory-sanitizer... " >&6; } - -# Check whether --with-memory_sanitizer was given. -if test "${with_memory_sanitizer+set}" = set; then : - withval=$with_memory_sanitizer; -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 -$as_echo "$withval" >&6; } -BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" -LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" -# MSan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-undefined-behavior-sanitizer" >&5 -$as_echo_n "checking for --with-undefined-behavior-sanitizer... " >&6; } - -# Check whether --with-undefined_behavior_sanitizer was given. -if test "${with_undefined_behavior_sanitizer+set}" = set; then : - withval=$with_undefined_behavior_sanitizer; -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 -$as_echo "$withval" >&6; } -BASECFLAGS="-fsanitize=undefined $BASECFLAGS" -LDFLAGS="-fsanitize=undefined $LDFLAGS" - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - # Most SVR4 platforms (e.g. Solaris) need -lsocket and -lnsl. { $as_echo "$as_me:${as_lineno-$LINENO}: checking for t_open in -lnsl" >&5 $as_echo_n "checking for t_open in -lnsl... " >&6; } diff --git a/configure.ac b/configure.ac index 3caff87dbd1b0c..e5783c8641e2c4 100644 --- a/configure.ac +++ b/configure.ac @@ -2595,6 +2595,47 @@ case $ac_sys_system/$ac_sys_release in ;; esac +AC_MSG_CHECKING(for --with-address-sanitizer) +AC_ARG_WITH(address_sanitizer, + AS_HELP_STRING([--with-address-sanitizer], + [enable AddressSanitizer memory error detector, 'asan' (default is no)]), +[ +AC_MSG_RESULT($withval) +BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="-fsanitize=address $LDFLAGS" +# ASan works by controlling memory allocation, our own malloc interferes. +with_pymalloc="no" +], +[AC_MSG_RESULT(no)]) + +AC_MSG_CHECKING(for --with-memory-sanitizer) +AC_ARG_WITH(memory_sanitizer, + AS_HELP_STRING([--with-memory-sanitizer], + [enable MemorySanitizer allocation error detector, 'msan' (default is no)]), +[ +AC_MSG_RESULT($withval) +BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" +# MSan works by controlling memory allocation, our own malloc interferes. +with_pymalloc="no" +], +[AC_MSG_RESULT(no)]) + +AC_MSG_CHECKING(for --with-undefined-behavior-sanitizer) +AC_ARG_WITH(undefined_behavior_sanitizer, + AS_HELP_STRING([--with-undefined-behavior-sanitizer], + [enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no)]), +[ +AC_MSG_RESULT($withval) +BASECFLAGS="-fsanitize=undefined $BASECFLAGS" +LDFLAGS="-fsanitize=undefined $LDFLAGS" +with_ubsan="yes" +], +[ +AC_MSG_RESULT(no) +with_ubsan="no" +]) + # Set info about shared libraries. AC_SUBST(SHLIB_SUFFIX) AC_SUBST(LDSHARED) @@ -2800,7 +2841,16 @@ then # to ensure that tests don't crash # Note: This matches the value of THREAD_STACK_SIZE in # thread_pthread.h - LINKFORSHARED="-Wl,-stack_size,4000000 $LINKFORSHARED" + stack_size="1000000" + if test "$with_ubsan" == "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" + AC_DEFINE_UNQUOTED(THREAD_STACK_SIZE, + 0x$stack_size, + [Custom thread stack size to fit the undefined behavior sanitizer runtime.]) + fi + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" if test "$enable_framework" then @@ -3044,43 +3094,6 @@ esac AC_MSG_RESULT("$TZPATH")]) AC_SUBST(TZPATH) -AC_MSG_CHECKING(for --with-address-sanitizer) -AC_ARG_WITH(address_sanitizer, - AS_HELP_STRING([--with-address-sanitizer], - [enable AddressSanitizer memory error detector, 'asan' (default is no)]), -[ -AC_MSG_RESULT($withval) -BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" -LDFLAGS="-fsanitize=address $LDFLAGS" -# ASan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" -], -[AC_MSG_RESULT(no)]) - -AC_MSG_CHECKING(for --with-memory-sanitizer) -AC_ARG_WITH(memory_sanitizer, - AS_HELP_STRING([--with-memory-sanitizer], - [enable MemorySanitizer allocation error detector, 'msan' (default is no)]), -[ -AC_MSG_RESULT($withval) -BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" -LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" -# MSan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" -], -[AC_MSG_RESULT(no)]) - -AC_MSG_CHECKING(for --with-undefined-behavior-sanitizer) -AC_ARG_WITH(undefined_behavior_sanitizer, - AS_HELP_STRING([--with-undefined-behavior-sanitizer], - [enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no)]), -[ -AC_MSG_RESULT($withval) -BASECFLAGS="-fsanitize=undefined $BASECFLAGS" -LDFLAGS="-fsanitize=undefined $LDFLAGS" -], -[AC_MSG_RESULT(no)]) - # Most SVR4 platforms (e.g. Solaris) need -lsocket and -lnsl. AC_CHECK_LIB(nsl, t_open, [LIBS="-lnsl $LIBS"]) # SVR4 AC_CHECK_LIB(socket, socket, [LIBS="-lsocket $LIBS"], [], $LIBS) # SVR4 sockets diff --git a/pyconfig.h.in b/pyconfig.h.in index 63438d857a070c..8c207f7f817753 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1509,6 +1509,10 @@ (which you can't on SCO ODT 3.0). */ #undef SYS_SELECT_WITH_SYS_TIME +/* Custom thread stack size to fit the undefined behavior sanitizer runtime. + */ +#undef THREAD_STACK_SIZE + /* Library needed by timemodule.c: librt may be needed for clock_gettime() */ #undef TIMEMODULE_LIB From 5d734c3fc324b1a93d39869b291341d646cc349c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 27 Aug 2021 17:00:32 +0200 Subject: [PATCH 3/4] add Blurb --- .../NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst diff --git a/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst b/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst new file mode 100644 index 00000000000000..29a6ff92554e1c --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst @@ -0,0 +1,3 @@ +When building CPython on macOS with ``./configure +--with-undefined-behavior-sanitizer --with-pydebug``, the stack size is now +quadrupled to allow for the entire test suite to pass. From 7044e8f31fe47688ed873a3bbd66e85dc5275d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 28 Aug 2021 12:52:06 +0200 Subject: [PATCH 4/4] Respond to Ronald's review --- Python/thread_pthread.h | 13 ++++++------- configure | 12 ++++++------ configure.ac | 14 +++++++------- pyconfig.h.in | 3 +-- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index ec7d737518b68c..a45d842ffe73d2 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -32,18 +32,17 @@ #define THREAD_STACK_SIZE 0 /* use default stack size */ #endif -/* The default stack size for new threads on OSX and BSD is small enough that +/* The default stack size for new threads on BSD is small enough that * we'll get hard crashes instead of 'maximum recursion depth exceeded' * exceptions. * - * The default stack sizes below are the empirically determined minimal stack + * The default stack size below is the empirically determined minimal stack * sizes where a simple recursive function doesn't cause a hard crash. + * + * For macOS the value of THREAD_STACK_SIZE is determined in configure.ac + * as it also depends on the other configure options like chosen sanitizer + * runtimes. */ -#if defined(__APPLE__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0 -#undef THREAD_STACK_SIZE -/* Note: This matches the value of -Wl,-stack_size in configure.ac */ -#define THREAD_STACK_SIZE 0x1000000 -#endif #if defined(__FreeBSD__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0 #undef THREAD_STACK_SIZE #define THREAD_STACK_SIZE 0x400000 diff --git a/configure b/configure index baa90dc98dd510..4f12972540d5f2 100755 --- a/configure +++ b/configure @@ -9871,20 +9871,20 @@ then # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - # Note: This matches the value of THREAD_STACK_SIZE in - # thread_pthread.h - stack_size="1000000" + stack_size="1000000" # 16 MB if test "$with_ubsan" == "yes" then # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" + stack_size="4000000" # 64 MB + fi + + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + cat >>confdefs.h <<_ACEOF #define THREAD_STACK_SIZE 0x$stack_size _ACEOF - fi - LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" if test "$enable_framework" then diff --git a/configure.ac b/configure.ac index e5783c8641e2c4..e01e0c13fba19c 100644 --- a/configure.ac +++ b/configure.ac @@ -2839,19 +2839,19 @@ then # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - # Note: This matches the value of THREAD_STACK_SIZE in - # thread_pthread.h - stack_size="1000000" + stack_size="1000000" # 16 MB if test "$with_ubsan" == "yes" then # Undefined behavior sanitizer requires an even deeper stack - stack_size="4000000" - AC_DEFINE_UNQUOTED(THREAD_STACK_SIZE, - 0x$stack_size, - [Custom thread stack size to fit the undefined behavior sanitizer runtime.]) + stack_size="4000000" # 64 MB fi + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + AC_DEFINE_UNQUOTED(THREAD_STACK_SIZE, + 0x$stack_size, + [Custom thread stack size depending on chosen sanitizer runtimes.]) + if test "$enable_framework" then LINKFORSHARED="$LINKFORSHARED "'$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)' diff --git a/pyconfig.h.in b/pyconfig.h.in index 8c207f7f817753..49407ab62b417d 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1509,8 +1509,7 @@ (which you can't on SCO ODT 3.0). */ #undef SYS_SELECT_WITH_SYS_TIME -/* Custom thread stack size to fit the undefined behavior sanitizer runtime. - */ +/* Custom thread stack size depending on chosen sanitizer runtimes. */ #undef THREAD_STACK_SIZE /* Library needed by timemodule.c: librt may be needed for clock_gettime() */