From dd759070a5907127b7cc169b4135228275c2f5ba Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 1 Nov 2021 19:41:49 +0100 Subject: [PATCH 1/7] ci_build.sh: guess the available amount of parallelism for the current system; consider load averages for GNU Make too --- ci_build.sh | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/ci_build.sh b/ci_build.sh index f56c529ddb..e35c98af79 100755 --- a/ci_build.sh +++ b/ci_build.sh @@ -75,6 +75,38 @@ esac [ -n "$MAKE" ] || MAKE=make [ -n "$GGREP" ] || GGREP=grep +# Set up the parallel make with reasonable limits, using several ways to +# gather and calculate this information. Note that "psrinfo" count is not +# an honest approach (there may be limits of current CPU set etc.) but is +# a better upper bound than nothing... +[ -n "$NCPUS" ] || { \ + NCPUS="`/usr/bin/getconf _NPROCESSORS_ONLN`" || \ + NCPUS="`/usr/bin/getconf NPROCESSORS_ONLN`" || \ + NCPUS="`cat /proc/cpuinfo | grep -wc processor`" || \ + { [ -x /usr/sbin/psrinfo ] && NCPUS="`/usr/sbin/psrinfo | wc -l`"; } \ + || NCPUS=1; } 2>/dev/null +[ x"$NCPUS" != x -a "$NCPUS" -ge 1 ] || NCPUS=1 + +[ x"$NPARMAKES" = x ] && { NPARMAKES="`expr "$NCPUS" '*' 2`" || NPARMAKES=2; } +[ x"$NPARMAKES" != x -a "$NPARMAKES" -ge 1 ] || NPARMAKES=2 +[ x"$MAXPARMAKES" != x ] && [ "$MAXPARMAKES" -ge 1 ] && \ + [ "$NPARMAKES" -gt "$MAXPARMAKES" ] && \ + echo "INFO: Detected or requested NPARMAKES=$NPARMAKES," \ + "however a limit of MAXPARMAKES=$MAXPARMAKES was configured" && \ + NPARMAKES="$MAXPARMAKES" + +# GNU make allows to limit spawning of jobs by load average of the host, +# where LA is (roughly) the average amount over the last {timeframe} of +# queued processes that are ready to compute but must wait for CPU. +# The rough estimate for VM builders however seems that they always have +# some non-trivial LA, so we set the default limit per CPU relatively high. +[ x"$PARMAKE_LA_LIMIT" = x ] && PARMAKE_LA_LIMIT="`expr $NCPUS '*' 8`".0 + +PARMAKE_FLAGS="-j $NPARMAKES" +if "$MAKE" --version 2>&1 | egrep 'GNU Make|Free Software Foundation' > /dev/null ; then + PARMAKE_FLAGS="$PARMAKE_FLAGS -l $PARMAKE_LA_LIMIT" +fi + # CI builds on Jenkins [ -z "$NODE_LABELS" ] || \ for L in $NODE_LABELS ; do @@ -201,7 +233,7 @@ configure_nut() { build_to_only_catch_errors() { ( echo "`date`: Starting the parallel build attempt (quietly to build what we can)..."; \ - $CI_TIME $MAKE VERBOSE=0 -k -j 8 all >/dev/null 2>&1 && echo "`date`: SUCCESS" ; ) || \ + $CI_TIME $MAKE VERBOSE=0 -k $PARMAKE_FLAGS all >/dev/null 2>&1 && echo "`date`: SUCCESS" ; ) || \ ( echo "`date`: Starting the sequential build attempt (to list remaining files with errors considered fatal for this build configuration)..."; \ $CI_TIME $MAKE VERBOSE=1 all -k ) || return $? @@ -751,7 +783,7 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-sp esac ( echo "`date`: Starting the parallel build attempt..."; \ - $CI_TIME $MAKE VERBOSE=1 -k -j 8 all; ) || \ + $CI_TIME $MAKE VERBOSE=1 -k $PARMAKE_FLAGS all; ) || \ ( echo "`date`: Starting the sequential build attempt..."; \ $CI_TIME $MAKE VERBOSE=1 all ) From 11aff9d4a8f2beee6b95e3b91cf91aac7524dcba Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 1 Nov 2021 19:45:01 +0100 Subject: [PATCH 2/7] ci_build.sh: enable parallel builds for "default-tgt:*" which should now pass well --- ci_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_build.sh b/ci_build.sh index e35c98af79..11854a0ff1 100755 --- a/ci_build.sh +++ b/ci_build.sh @@ -658,7 +658,7 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-sp # that include DISTCHECK_FLAGS if provided DISTCHECK_FLAGS="`for F in "${CONFIG_OPTS[@]}" ; do echo "'$F' " ; done | tr '\n' ' '`" export DISTCHECK_FLAGS - $CI_TIME $MAKE VERBOSE=1 DISTCHECK_FLAGS="$DISTCHECK_FLAGS" "$BUILD_TGT" + $CI_TIME $MAKE VERBOSE=1 DISTCHECK_FLAGS="$DISTCHECK_FLAGS" $PARMAKE_FLAGS "$BUILD_TGT" echo "=== Are GitIgnores good after '$MAKE $BUILD_TGT'? (should have no output below)" git status -s || true From 0b49a27ddf582d69697bfee441b4b826fd3a6561 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 1 Nov 2021 19:48:05 +0100 Subject: [PATCH 3/7] configure.ac: test more proactively if valgrind can test things and not crash by itself --- configure.ac | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index cb0a761d22..f953286052 100644 --- a/configure.ac +++ b/configure.ac @@ -1887,7 +1887,11 @@ dnl we support an explicit --without-valgrind (aka --with-valgrind=no) too. AS_IF([test -n "${VALGRIND}"], [ AC_MSG_CHECKING([whether valgrind is usable on current platform]) AS_IF([( ${VALGRIND} --help >/dev/null 2>/dev/null )], - [AC_MSG_RESULT([yes])], + [AS_IF([( ${VALGRIND} /bin/sh -c true >/dev/null 2>/dev/null )], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + VALGRIND="none" + ])], [AC_MSG_RESULT([no]) VALGRIND="none" ]) From 31443f845d2ba7192e65a56d1ff28266015c5405 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 1 Nov 2021 19:51:14 +0100 Subject: [PATCH 4/7] ci_build.sh: report chosen settings for parallel builds --- ci_build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci_build.sh b/ci_build.sh index 11854a0ff1..c7c1dc9513 100755 --- a/ci_build.sh +++ b/ci_build.sh @@ -105,6 +105,9 @@ esac PARMAKE_FLAGS="-j $NPARMAKES" if "$MAKE" --version 2>&1 | egrep 'GNU Make|Free Software Foundation' > /dev/null ; then PARMAKE_FLAGS="$PARMAKE_FLAGS -l $PARMAKE_LA_LIMIT" + echo "Parallel builds would spawn up to $NPARMAKES jobs (detected $NCPUS CPUs), or peak out at $PARMAKE_LA_LIMIT system load average" >&2 +else + echo "Parallel builds would spawn up to $NPARMAKES jobs (detected $NCPUS CPUs)" >&2 fi # CI builds on Jenkins From 11ecf167f87e41c22c9bf869bbfd12334b1d8d7d Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 1 Nov 2021 19:57:47 +0100 Subject: [PATCH 5/7] ci_build.sh: enable parallel builds for "distcheck" in default builds, which should now pass well (after usual build and check passed) --- ci_build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ci_build.sh b/ci_build.sh index c7c1dc9513..2a1e6f7f73 100755 --- a/ci_build.sh +++ b/ci_build.sh @@ -681,8 +681,10 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-sp [ -z "$CI_TIME" ] || echo "`date`: Trying to spellcheck documentation of the currently tested project..." # Note: use the root Makefile's spellcheck recipe which goes into # sub-Makefiles known to check corresponding directory's doc files. + # Note: no PARMAKE_FLAGS here - better have this output readably + # ordered in case of issues (in sequential replay below). ( echo "`date`: Starting the quiet build attempt for target $BUILD_TYPE..." >&2 - $CI_TIME $MAKE -s VERBOSE=0 SPELLCHECK_ERROR_FATAL=yes -k spellcheck >/dev/null 2>&1 \ + $CI_TIME $MAKE -s VERBOSE=0 SPELLCHECK_ERROR_FATAL=yes -k $PARMAKE_FLAGS spellcheck >/dev/null 2>&1 \ && echo "`date`: SUCCEEDED the spellcheck" >&2 ) || \ ( echo "`date`: FAILED something in spellcheck above; re-starting a verbose build attempt to summarize:" >&2 @@ -699,6 +701,8 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-sp ### Still, there remains value in also checking the script syntax ### by the very version of the shell interpreter that would run ### these scripts in production usage of the resulting packages. + ### Note: no PARMAKE_FLAGS here - better have this output readably + ### ordered in case of issues. ( $CI_TIME $MAKE VERBOSE=1 shellcheck check-scripts-syntax ) exit $? ;; @@ -814,7 +818,7 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-sp # that include DISTCHECK_FLAGS if provided DISTCHECK_FLAGS="`for F in "${CONFIG_OPTS[@]}" ; do echo "'$F' " ; done | tr '\n' ' '`" export DISTCHECK_FLAGS - $CI_TIME $MAKE VERBOSE=1 DISTCHECK_FLAGS="$DISTCHECK_FLAGS" distcheck + $CI_TIME $MAKE VERBOSE=1 DISTCHECK_FLAGS="$DISTCHECK_FLAGS" $PARMAKE_FLAGS distcheck echo "=== Are GitIgnores good after '$MAKE distcheck'? (should have no output below)" git status -s || true From ac30789d375f42befd1c84dedfb825d4c5b93ab5 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 1 Nov 2021 19:58:18 +0100 Subject: [PATCH 6/7] ci_build.sh: enable parallel builds for default build (one without a BUILD_TYPE specified) --- ci_build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci_build.sh b/ci_build.sh index 2a1e6f7f73..2fedadf588 100755 --- a/ci_build.sh +++ b/ci_build.sh @@ -844,7 +844,9 @@ bindings) ./autogen.sh #./configure ./configure --with-cgi=auto --with-serial=auto --with-dev=auto --with-doc=skip - $MAKE all && $MAKE check + #$MAKE all && \ + $MAKE $PARMAKE_FLAGS all && \ + $MAKE check ;; *) pushd "./builds/${BUILD_TYPE}" && REPO_DIR="$(dirs -l +1)" ./ci_build.sh From 111f33390209a5c975315d2c1d30bf1639e60710 Mon Sep 17 00:00:00 2001 From: Jim Klimov Date: Mon, 1 Nov 2021 20:02:51 +0100 Subject: [PATCH 7/7] ci_build.sh: give the caller a way to disable parallel builds with PARMAKE_FLAGS=" " (space) --- ci_build.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ci_build.sh b/ci_build.sh index 2fedadf588..8f29b81626 100755 --- a/ci_build.sh +++ b/ci_build.sh @@ -102,12 +102,16 @@ esac # some non-trivial LA, so we set the default limit per CPU relatively high. [ x"$PARMAKE_LA_LIMIT" = x ] && PARMAKE_LA_LIMIT="`expr $NCPUS '*' 8`".0 -PARMAKE_FLAGS="-j $NPARMAKES" -if "$MAKE" --version 2>&1 | egrep 'GNU Make|Free Software Foundation' > /dev/null ; then - PARMAKE_FLAGS="$PARMAKE_FLAGS -l $PARMAKE_LA_LIMIT" - echo "Parallel builds would spawn up to $NPARMAKES jobs (detected $NCPUS CPUs), or peak out at $PARMAKE_LA_LIMIT system load average" >&2 -else - echo "Parallel builds would spawn up to $NPARMAKES jobs (detected $NCPUS CPUs)" >&2 +# After all the tunable options above, this is the one which takes effect +# for actual builds with parallel phases. Specify a whitespace to neuter. +if [ -z "$PARMAKE_FLAGS" ]; then + PARMAKE_FLAGS="-j $NPARMAKES" + if LANG=C LC_ALL=C "$MAKE" --version 2>&1 | egrep 'GNU Make|Free Software Foundation' > /dev/null ; then + PARMAKE_FLAGS="$PARMAKE_FLAGS -l $PARMAKE_LA_LIMIT" + echo "Parallel builds would spawn up to $NPARMAKES jobs (detected $NCPUS CPUs), or peak out at $PARMAKE_LA_LIMIT system load average" >&2 + else + echo "Parallel builds would spawn up to $NPARMAKES jobs (detected $NCPUS CPUs)" >&2 + fi fi # CI builds on Jenkins