Skip to content

Commit 333b806

Browse files
jimklimovjimklimov
authored
Merge a5488fd into 90b0476 
2 parents 90b0476 + a5488fd commit 333b806

4 files changed

Lines changed: 105 additions & 36 deletions

File tree

‎.circleci/config.yml‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -171,16 +171,20 @@ jobs:
171171
./ci_build.sh
172172
173173
- run:
174-
name: "ccache stats after initial build"
174+
name: "ccache stats after build and checks"
175175
command: ccache -s || true
176176

177-
- run:
178-
name: "verify parallel build recipes in subdirs"
179-
command: make -s -j check-parallel-builds || exit
177+
# - run:
178+
# name: "ccache stats after initial build"
179+
# command: ccache -s || true
180180

181-
- run:
182-
name: "ccache stats after a few rebuilds"
183-
command: ccache -s || true
181+
# - run:
182+
# name: "verify parallel build recipes in subdirs"
183+
# command: make -s -j check-parallel-builds || exit
184+
185+
# - run:
186+
# name: "ccache stats after a few rebuilds"
187+
# command: ccache -s || true
184188

185189
# NOTE: Detailed key name allows every scenario to only track its
186190
# own ccache objects, which makes sense for different compilers

‎Makefile.am‎

Lines changed: 29 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -697,17 +697,22 @@ spellcheck spellcheck-interactive:
697697

698698
# Auto-parallel recipe (if current 'make' implementation supports the "-j N"
699699
# syntax; the optional MAXPARMAKES may be set in NUT CI farm style builds):
700-
spellcheck-quick:
701-
+@case " $(MAKEFLAGS) $(AM_MAKEFLAGS)" in \
702-
*"j"*) $(MAKE) $(AM_MAKEFLAGS) -k -s spellcheck && exit ;; \
700+
SET_PARMAKES_OPT = \
701+
+@PARMAKES_OPT=""; \
702+
case " $(MAKEFLAGS) $(AM_MAKEFLAGS)" in \
703+
*"j"*) ;; \
703704
*) \
704705
if ! [ "$${MAXPARMAKES-}" -gt 1 ] 2>/dev/null ; then \
705706
MAXPARMAKES=8 ; \
706707
fi ; \
707-
$(MAKE) $(AM_MAKEFLAGS) -k -s -j $${MAXPARMAKES} spellcheck \
708-
&& exit ;; \
708+
PARMAKES_OPT="-j $${MAXPARMAKES}" ; \
709+
;; \
709710
esac
710711

712+
spellcheck-quick:
713+
+@$(SET_PARMAKES_OPT); \
714+
$(MAKE) $(AM_MAKEFLAGS) -k -s ${PARMAKES_OPT} spellcheck
715+
711716
# Run auto-parallel recipe, and if something fails - re-run interactively:
712717
spellcheck-interactive-quick:
713718
+@$(MAKE) $(AM_MAKEFLAGS) -k -s spellcheck-quick && exit ; \
@@ -1444,35 +1449,37 @@ check-files-quick: $(CHECK_FILES_QUICK_TARGETS)
14441449
# Autotools hook: run the quick checks before recursing for defaults:
14451450
check-recursive: check-files-quick
14461451

1452+
# Caller can set this to "false" to not regenerate below
1453+
# (e.g. in CI vs. developer iterations):
1454+
CHECK_PARALLEL_BUILDS_REGEN = true
1455+
14471456
# Not pulled in directly so far, can be used by developers to verify that build
14481457
# rules (inlcuding dependencies pulled from other directories) make sense and
14491458
# do not cause conflict by writing into same file names. Something that builds
14501459
# from root directory arrange nicely, but needs careful balancing on a tightrope
14511460
# if developers build pieces of code right in the directory of their interest.
14521461
# It is recommended to have "ccache" (or similar) setup working, to minimize the
14531462
# time cost and workload of compilations involved in these looped build retries.
1463+
# Oddly, the NetBSD 9.2 `bmake` faced with out-of-tree (subdir) builds does
1464+
# find itself work to do even if called in the root dir (with no `Makefile`
1465+
# generated there -- it detects a build to do under e.g. `obj/` in CI tests),
1466+
# but also tends to have that directory as current at the time we try to call
1467+
# the `./config.status` line -- and fails there. Hence the `cd top_builddir`.
14541468
# Auto-parallel recipe (if current 'make' implementation supports the "-j N"
14551469
# syntax; the optional MAXPARMAKES may be set in NUT CI farm style builds):
14561470
check-parallel-builds:
1457-
automake -f
1458-
./config.status
1459-
+@MAXPARMAKES_OPT=""; \
1460-
case " $(MAKEFLAGS) $(AM_MAKEFLAGS)" in \
1461-
*"j"*) ;; \
1462-
*) \
1463-
if ! [ "$${MAXPARMAKES-}" -gt 1 ] 2>/dev/null ; then \
1464-
MAXPARMAKES=8 ; \
1465-
fi ; \
1466-
MAXPARMAKES_OPT="-j $${MAXPARMAKES}" ; \
1467-
;; \
1468-
esac ; \
1471+
if [ x"$(CHECK_PARALLEL_BUILDS_REGEN)" = xtrue ] ; then cd '$(top_srcdir)' && $(AUTOMAKE) -f ; fi
1472+
if [ x"$(CHECK_PARALLEL_BUILDS_REGEN)" = xtrue ] ; then cd '$(top_builddir)' && ./config.status ; fi
1473+
+@$(SET_PARMAKES_OPT); \
14691474
DIRS=""; \
1470-
for D in `find . -name '*.c' -o -name '*.cpp' | sed 's,/[^/]*\.cp*$$,,' | uniq` ; do \
1471-
[ -e "$$D/Makefile.am" ] && [ -s "$$D/Makefile" ] || continue ; \
1472-
$(MAKE) $(AM_MAKEFLAGS) -k -s $${MAXPARMAKES_OPT} clean || { RES=$$?; echo "$@: FAILED: make clean before going to $$D" >&2; exit $$RES; } ; \
1475+
cd '$(abs_top_builddir)' || exit ; \
1476+
$(MAKE) $(AM_MAKEFLAGS) -k -s $${PARMAKES_OPT} clean || { RES=$$?; echo "$@: FAILED: make pre-clean before checking subdirs for sources" >&2; exit $$RES; } ; \
1477+
for D in `cd '$(top_srcdir)' && find . -name '*.c' -o -name '*.cpp' | sed 's,/[^/]*\.cp*$$,,' | uniq` ; do \
1478+
[ -e "$(top_srcdir)/$$D/Makefile.am" ] && [ -s "$$D/Makefile" ] || continue ; \
1479+
$(MAKE) $(AM_MAKEFLAGS) -k -s $${PARMAKES_OPT} clean || { RES=$$?; echo "$@: FAILED: make clean before going to $$D" >&2; exit $$RES; } ; \
14731480
echo " $@ in $${D}" ; \
1474-
( cd "$$D" && $(MAKE) $(AM_MAKEFLAGS) -k -s $${MAXPARMAKES_OPT} ) || { RES=$$?; echo "$@: FAILED: parallel make in $$D" >&2; \
1475-
echo "To investigate, try: $(MAKE) $(MAKEFLAGS) $(AM_MAKEFLAGS) $${MAXPARMAKES_OPT} clean ; automake -f && ./config.status && clear && (cd $${D}/ && $(MAKE) $(MAKEFLAGS) $(AM_MAKEFLAGS) V=1 $${MAXPARMAKES_OPT} 2>&1 ; echo $?) | tee /tmp/make.log ; grep -E '(\] Error|No rule to)' /tmp/make.log && less /tmp/make.log"; \
1481+
( cd "$$D" && $(MAKE) $(AM_MAKEFLAGS) -k -s $${PARMAKES_OPT} ) || { RES=$$?; echo "$@: FAILED: parallel make in $$D" >&2; \
1482+
echo "To investigate, try: (MAKEFLAGS='$(MAKEFLAGS)' ; export MAKEFLAGS; $(MAKE) $(AM_MAKEFLAGS) $${PARMAKES_OPT} clean ; automake -f && ./config.status && clear && (cd $${D}/ && $(MAKE) $(AM_MAKEFLAGS) V=1 $${PARMAKES_OPT} 2>&1 ; echo $$?) | tee /tmp/make.log ; RES=$$? ; grep -E '(\] Error|No rule to)' /tmp/make.log && less /tmp/make.log ; exit $$RES )"; \
14761483
echo "If builds were interrupted before, you may also have to re-initialize the build area completely, e.g.: git clean -fdX ; ./ci_build.sh && $(MAKE) $(MAKEFLAGS) $(AM_MAKEFLAGS) $@" ; \
14771484
exit $$RES; } ; \
14781485
DIRS="$${DIRS} $${D}" ; \

‎ci_build.sh‎

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -513,9 +513,14 @@ fi
513513
# For two-phase builds (quick parallel make first, sequential retry if failed)
514514
# how verbose should that first phase be? Nothing, automake list of ops, CLIs?
515515
# See build_to_only_catch_errors_target() for a consumer of this setting.
516+
# CI_PARMAKE_VERBOSITY_CPB is for "check-parallel-builds" part below.
516517
case "${CI_PARMAKE_VERBOSITY-}" in
517-
silent|quiet|verbose|default) ;;
518-
*) CI_PARMAKE_VERBOSITY=silent ;;
518+
silent|quiet|verbose|default)
519+
[ -n "${CI_PARMAKE_VERBOSITY_CPB-}" ] || CI_PARMAKE_VERBOSITY_CPB="${CI_PARMAKE_VERBOSITY-}"
520+
;;
521+
*) CI_PARMAKE_VERBOSITY=silent
522+
[ -n "${CI_PARMAKE_VERBOSITY_CPB-}" ] || CI_PARMAKE_VERBOSITY_CPB=quiet
523+
;;
519524
esac
520525

521526
# Set up the parallel make with reasonable limits, using several ways to
@@ -636,6 +641,11 @@ for L in $NODE_LABELS ; do
636641
[ -n "$CANBUILD_WITH_LIBLTDL" ] || CANBUILD_WITH_LIBLTDL=no ;;
637642
"NUT_BUILD_CAPS=libltdl"|"NUT_BUILD_CAPS=libltdl=yes")
638643
[ -n "$CANBUILD_WITH_LIBLTDL" ] || CANBUILD_WITH_LIBLTDL=yes ;;
644+
645+
# For now like this; VM systems with a clock skew can have natural
646+
# problems with parallel tasks, which we can not do much about.
647+
"NUT_BUILD_CAPS=check-parallel-builds=no")
648+
[ -n "${CI_DO_CHECK_PARALLEL_BUILDS-}" ] || CI_DO_CHECK_PARALLEL_BUILDS=false ;;
639649
esac
640650
done
641651

@@ -1151,7 +1161,7 @@ build_to_only_catch_errors_target() {
11511161
$CI_TIME $MAKE $MAKE_FLAGS_QUIET -k $PARMAKE_FLAGS "$@" >/dev/null ;;
11521162
quiet)
11531163
$CI_TIME $MAKE $MAKE_FLAGS_QUIET -k $PARMAKE_FLAGS "$@" ;;
1154-
silent)
1164+
verbose)
11551165
$CI_TIME $MAKE $MAKE_FLAGS_VERBOSE -k $PARMAKE_FLAGS "$@" ;;
11561166
default)
11571167
$CI_TIME $MAKE -k $PARMAKE_FLAGS "$@" ;;
@@ -1945,6 +1955,16 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-al
19451955
echo "NOTE: adapted BUILD_TYPE 'default-all-errors' => '${BUILD_TYPE}'" >&2
19461956
fi
19471957

1958+
if [ x"${HAVE_CCACHE-}" = xyes ] && [ x"${CI_DO_CHECK_PARALLEL_BUILDS-}" = x ] ; then
1959+
# Ideally massive-build CI recipes would set this option
1960+
# for scenarios they are interested in, e.g. once per OS
1961+
# and make implementation, not all hundreds of builds?..
1962+
# On the other hand, catching issues (race conditions in
1963+
# recipes) is a big-numbers game...
1964+
CI_DO_CHECK_PARALLEL_BUILDS=true
1965+
echo "NOTE: we have ccache, so enabled 'make check-parallel-builds' for combos below" >&2
1966+
fi
1967+
19481968
# Try to run various build scenarios to collect build errors
19491969
# (no checks here) as configured further by caller's choice
19501970
# of BUILD_WARNFATAL and/or BUILD_WARNOPT envvars above.
@@ -2298,6 +2318,12 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-al
22982318
;;
22992319
esac
23002320

2321+
# Snippet from autogen.sh: restore files required by autoconf
2322+
# for non-"foreign" projects that a deep clean in other loops
2323+
# could have destroyed:
2324+
[ -f "${SCRIPTDIR}/NEWS" ] || { echo "Please see NEWS.adoc for actual contents" > "${SCRIPTDIR}/NEWS"; }
2325+
[ -f "${SCRIPTDIR}/README" ] || { echo "Please see README.adoc for actual contents" > "${SCRIPTDIR}/README"; }
2326+
23012327
configure_nut
23022328
) || {
23032329
RES_ALLERRORS=$?
@@ -2340,6 +2366,35 @@ default|default-alldrv|default-alldrv:no-distcheck|default-all-errors|default-al
23402366
fi
23412367
}
23422368

2369+
# Check that the current MAKE implementation deals with parallel
2370+
# recipes properly. Ideally this is sped up by ccache. At least
2371+
# privately CI_FAILFAST=true to not retry this one sequentially.
2372+
# Note two passings of CHECK_PARALLEL_BUILDS_REGEN=false - as an
2373+
# envvar and as a make argument, to ensure that different `make`
2374+
# implementations honour our desire.
2375+
# WARNING: This check does repetitively `make clean` along the
2376+
# way, so should be the last operation before scenario clean-up!
2377+
if [ x"$CI_DO_CHECK_PARALLEL_BUILDS" = xtrue ] ; then
2378+
CI_FAILFAST=true \
2379+
CI_PARMAKE_VERBOSITY="${CI_PARMAKE_VERBOSITY_CPB-}" \
2380+
CHECK_PARALLEL_BUILDS_REGEN=false \
2381+
build_to_only_catch_errors_target \
2382+
CHECK_PARALLEL_BUILDS_REGEN=false \
2383+
check-parallel-builds \
2384+
&& {
2385+
SUCCEEDED+=("TESTCOMBO=${TESTCOMBO}[check-parallel-builds]")
2386+
} || {
2387+
RES_ALLERRORS=$?
2388+
FAILED+=("TESTCOMBO=${TESTCOMBO}[check-parallel-builds]")
2389+
# Help find end of build (before cleanup noise) in logs:
2390+
echo "=== FAILED 'TESTCOMBO=${TESTCOMBO}' check-parallel-builds"
2391+
if [ "$CI_FAILFAST" = true ]; then
2392+
echo "===== Aborting because CI_FAILFAST=$CI_FAILFAST" >&2
2393+
break
2394+
fi
2395+
}
2396+
fi
2397+
23432398
# Note: when `expr` calculates a zero value below, it returns
23442399
# an "erroneous" `1` as exit code. Why oh why?..
23452400
# (UPDATE: because expr returns boolean, and calculated 0 is false;

‎tools/nut-usbinfo.pl‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -287,13 +287,10 @@ sub find_usbdevs
287287
print stderr "find_usbdevs(): pwd='" . Cwd::cwd() . "' nameFile='" . $_ . "'\n";
288288
}
289289

290-
# maybe there's an option to turn off all .* files, but anyway this is stupid
291290
# Note that on some platforms "." and ".." do also pop up;
292291
# take care to NOT prune (avoid recursion into) the "." one:
293-
return $File::Find::prune = 1 if ($_ eq '.svn') || ($_ =~ /^\.#/) || ($_ =~ /\.(orig|o|la|lo|exe)$/) || ($_ eq '.libs') || ($_ eq '.deps') || ($_ eq '..');
294292
return $File::Find::prune = 0 if ($_ eq '.');
295-
# FIXME: Skip libtool wrappers or binary builds of drivers without extension
296-
# Maybe ONLY walk *.c and *.h files (and subdirs)?..
293+
return $File::Find::prune = 1 if ($_ eq '..') || ($_ =~ /^\.#/) || ($_ eq '.libs') || ($_ eq '.deps') || ($_ eq '.svn') || ($_ eq '.git');
297294

298295
if (-d $_) {
299296
# FIXME: in current NUT vanilla code we do not support subdirs
@@ -304,6 +301,12 @@ sub find_usbdevs
304301
return $File::Find::prune = 1;
305302
}
306303

304+
# To skip libtool wrappers or binary builds of drivers without
305+
# extension as well as temporary make files that appear and
306+
# dissipate (especially during parallel builds), ONLY walk
307+
# the *.c and *.h files (and subdirs):
308+
return $File::Find::prune = 1 if !($_ =~ /\.[ch]$/);
309+
307310
my $nameFile=$_;
308311
my $lastComment="";
309312

0 commit comments

Comments
 (0)