From b254a8637c59e6cee626b7433f82e9fbef278278 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Mon, 8 Feb 2021 18:28:54 +0000 Subject: [PATCH 01/19] xtensa-build-all.sh: add PATH to build log The script already logs the full CMake command that is re-usable outside this script... except for the PATH change. Expose that sneaky PATH change. Debugging every build issue starts with peeling the too many layers of indirection. Also fix some minor issue in the help message. Signed-off-by: Marc Herbert (cherry picked from commit 419806fb1c50516c8f84f590b7c3b12ad0da2dff) --- scripts/xtensa-build-all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/xtensa-build-all.sh b/scripts/xtensa-build-all.sh index 703de63740de..b07e799b8ae0 100755 --- a/scripts/xtensa-build-all.sh +++ b/scripts/xtensa-build-all.sh @@ -40,7 +40,7 @@ usage: $0 [options] platform(s) -u Force UP ARCH -d Enable debug build -c Interactive menuconfig - -o copies the file argument from src/arch/xtensa/configs/override/$arg.config + -o arg, copies src/arch/xtensa/configs/override/.config to the build directory after invoking CMake and before Make. -k Configure rimage to use a non-default \${RIMAGE_PRIVATE_KEY} DEPRECATED: use the more flexible \${PRIVATE_KEY_OPTION} below. @@ -347,6 +347,7 @@ do mkdir "$BUILD_DIR" cd "$BUILD_DIR" + printf 'PATH=%s\n' "$PATH" ( set -x # log the main commands and their parameters cmake -DTOOLCHAIN="$TOOLCHAIN" \ -DROOT_DIR="$ROOT" \ From f8143773a227aa58d036efe80fd56949ad0ad170 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 22 Jan 2021 07:53:30 +0000 Subject: [PATCH 02/19] Copy sof-$platform.{ri,ldc} to sof.{ri,ldc} As reported in #3491, there is confusion between platform names, CPU names, PCH names, toolchain names, signing schemes and what not. For instance in "build_apl_gcc/sof-apl.ri", the first "apl" matches the name of a defconfig file while the second "apl" matches the name of a signing scheme. When building out-of-source, there is no reason to vary the filenames of the output depending on the build configuration, changing the name of the build directory is enough. This simplifies automation logic including the next commit that adds a new install/GNUmakefile. Signed-off-by: Marc Herbert (cherry picked from commit 0e60ec8da49143e706ed686421d1b879a35a0a3a) --- src/arch/xtensa/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/arch/xtensa/CMakeLists.txt b/src/arch/xtensa/CMakeLists.txt index b429348e5808..ff10dd91a4e4 100644 --- a/src/arch/xtensa/CMakeLists.txt +++ b/src/arch/xtensa/CMakeLists.txt @@ -504,7 +504,9 @@ else() add_custom_target( bin ALL COMMAND ${CMAKE_COMMAND} -E copy sof-${fw_name}.ri ${PROJECT_BINARY_DIR}/sof-${fw_output_name}.ri + COMMAND ${CMAKE_COMMAND} -E copy sof-${fw_name}.ri ${PROJECT_BINARY_DIR}/sof.ri COMMAND ${CMAKE_COMMAND} -E copy sof-${fw_name}.ldc ${PROJECT_BINARY_DIR}/sof-${fw_output_name}.ldc + COMMAND ${CMAKE_COMMAND} -E copy sof-${fw_name}.ldc ${PROJECT_BINARY_DIR}/sof.ldc DEPENDS run_meu bin_extras glue_binary_files VERBATIM USES_TERMINAL From 9bf1e54587360cfce92918e4316e84f87c0b579d Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 22 Jan 2021 08:15:43 +0000 Subject: [PATCH 03/19] New install/GNUmakefile to gradually replace scripts in sof-bin.git More specifically replacing sof-bin/go.sh and sof-bin/publish.sh and also sof/scripts/sof-target-install.sh eventually. "make install" code has always belonged to source repositories because developers need to install too and we want everyone to use the same installers. It's also easier to have all the information in a single place. Once the layout in sof-bin mirrors the /lib/firmware/intel layout exactly, sof-bin does not need any installation code any more. Mixing source and binaries in the same repo is also a "code smell", notably because it forces branching them together. Using a higher level build tool for installation instead of plain scripts has a few benefits: - Multiple entry points: easy to invoke (and test) any part of the installer individually - ... while invoking dependencies automatically. - Other features "for free" like: - errexit - error messages like "dunno how to build file x" - commands are logged by default - Also gets rid of most of the large code duplication in go.sh and publish.sh, so: - Enabling or disabling a platform is a 3-character change - Allows platform selection in local config file (even just one platform) - Much harder to add inconsistencies - Much easier to review correctness, for instance no need to scrutinize every line to see which platforms are aliased. Signed-off-by: Marc Herbert (cherry picked from commit c6edc5f51e069e2c70c7cf0c50567ed916b3acd7) --- installer/.gitignore | 1 + installer/GNUmakefile | 204 +++++++++++++++++++++++++++++++++++++ installer/README.md | 78 ++++++++++++++ installer/sample-config.mk | 23 +++++ 4 files changed, 306 insertions(+) create mode 100644 installer/.gitignore create mode 100644 installer/GNUmakefile create mode 100644 installer/README.md create mode 100644 installer/sample-config.mk diff --git a/installer/.gitignore b/installer/.gitignore new file mode 100644 index 000000000000..24404a12cf22 --- /dev/null +++ b/installer/.gitignore @@ -0,0 +1 @@ +/staging/sof* diff --git a/installer/GNUmakefile b/installer/GNUmakefile new file mode 100644 index 000000000000..4b7d623c0a16 --- /dev/null +++ b/installer/GNUmakefile @@ -0,0 +1,204 @@ + +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation + + +.DEFAULT_GOAL := stage +.PHONY: clean stage rsync +.PHONY: signed unsigned ldicts topologies +.PHONY: compare signed_dummies + +# Override ?= variables in config.mk +-include config.mk + +UNSIGNED_list ?= bdw byt cht +SIGNED_list ?= apl cnl icl jsl tgl +# older SOF versions +# SIGNED_list += hsw kbl skl sue +ALIAS_list ?= glk cfl cml ehl + +$(info UNSIGNED_list = ${UNSIGNED_list} ) +$(info SIGNED_list = ${SIGNED_list} ) +$(info ALIAS_list = ${ALIAS_list} ) + +# Same code, same Intel key +target_of_glk := apl +target_of_cfl := cnl +target_of_cml := cnl + +# Same code, different Intel key +target_of_ehl := tgl + +TOOLCHAIN ?= gcc + +TREE_OPTS ?= --sort=size --dirsfirst +INSTALL_OPTS ?= -D -p -m 0664 + +# Keep SOF_VERSION optional + +SOF_VERSION ?= $(shell git describe --dirty) +ifneq (${SOF_VERSION},) +VERSION_DIR := ${SOF_VERSION}/ +VERSION_SUFFIX := -${SOF_VERSION} +endif + + + + ################################ + ### Top-level directories #### + ################################ + +# Our input: build_*_?cc/ directories +BUILDS_ROOT ?= ../ + +STAGING_SOF ?= staging/sof +STAGING_SOF_VERSION := ${STAGING_SOF}${VERSION_SUFFIX} + +STAGING_SOF_TPLG ?= staging/sof-tplg + +stage: signed unsigned ldicts aliases topologies +ifneq (${STAGING_SOF_VERSION},${STAGING_SOF}) + ln -sfT sof${VERSION_SUFFIX} ${STAGING_SOF} + test -e ${STAGING_SOF} +endif + @file ${STAGING_SOF} + @tree ${TREE_OPTS} ${STAGING_SOF_VERSION} + +COMMUNITY := ${STAGING_SOF_VERSION}/community +INTEL_SIGNED := ${STAGING_SOF_VERSION}/intel-signed +${COMMUNITY} ${INTEL_SIGNED}: + mkdir -p $@ + + + ##################################### + ### rsync to local or remote #### + ##################################### + +# Default value +FW_DESTDIR ?= /lib/firmware/intel/ +# We don't depend on any other target so: +# - it's possible to deploy a staging _subset_, +# - sudo never builds by accident +rsync: + rsync -a --info=progress2 staging/sof* "${FW_DESTDIR}" +ifneq (${USER_DESTDIR},) + # TODO: add more user space binaries: sof-ctl, probes,... + # absorbe scripts/sof-target-install.sh + # Requires ./scripts/build-tools.sh -l ... + rsync -a ${BUILDS_ROOT}/tools/build_tools/logger/sof-logger ${USER_DESTDIR} +endif + +clean: + ${RM} -r staging/sof* + + + ########################################################## + ### Stage sof-*.ri firmware files and symbolic links #### + ########################################################## + +# +# 1. Stages all *.ri files +# +# 2. Create symbolic links, including (broken) intel-signed symbolic +# links that must be fixed in a final release, otherwise the release +# is incomplete. To check all symlinks: file $(find -type l) +# + +# '%' is the platform name +SIGNED_FWS := ${SIGNED_list:%=${COMMUNITY}/sof-%.ri} +# $(info SIGNED_FWS = ${SIGNED_FWS}) +signed: ${SIGNED_FWS} +${SIGNED_FWS}: ${COMMUNITY}/sof-%.ri: \ + ${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ri \ + | ${COMMUNITY} ${INTEL_SIGNED} + install ${INSTALL_OPTS} $< $@ + ln -sfT intel-signed/sof-$*.ri ${STAGING_SOF_VERSION}/sof-$*.ri + +# '%' is the platform name +UNSIGNED_FWS := ${UNSIGNED_list:%=${STAGING_SOF_VERSION}/sof-%.ri} +# $(info UNSIGNED_FWS = ${UNSIGNED_FWS}) +unsigned: ${UNSIGNED_FWS} +${UNSIGNED_FWS}: ${STAGING_SOF_VERSION}/sof-%.ri: \ + ${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ri + install ${INSTALL_OPTS} $< $@ + + + ######################################## + ### Stage *.ldc logger dictionaries ### + ######################################## + +# '%' is the platform name +LDICTS := ${UNSIGNED_list:%=${STAGING_SOF_VERSION}/sof-%.ldc} \ + ${SIGNED_list:%=${STAGING_SOF_VERSION}/sof-%.ldc} +# $(info LDICTS = ${LDICTS}) +ldicts: ${LDICTS} +${LDICTS}: ${STAGING_SOF_VERSION}/sof-%.ldc: \ + ${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ldc + install ${INSTALL_OPTS} $< $@ + + + ####################################### + ### Platform -> platform aliases #### + ####################################### + +# '%' is the platform name +COMM_ALIASES := ${ALIAS_list:%=${STAGING_SOF_VERSION}/community/sof-%.ri} +DICT_ALIASES := ${ALIAS_list:%=${STAGING_SOF_VERSION}/sof-%.ldc} +SIGNED_ALIASES := ${ALIAS_list:%=${STAGING_SOF_VERSION}/sof-%.ri} +aliases: ${SIGNED_ALIASES} ${COMM_ALIASES} ${DICT_ALIASES} + +${COMM_ALIASES}: ${STAGING_SOF_VERSION}/community/sof-%.ri: + ln -sfT sof-${target_of_$*}.ri $@ + +${DICT_ALIASES}: ${STAGING_SOF_VERSION}/sof-%.ldc: + ln -sfT sof-${target_of_$*}.ldc $@ + +# Some have the same key, others just the code. We don't make any +# difference here and let the intel-signed/ directory handle this. +${SIGNED_ALIASES}: ${STAGING_SOF_VERSION}/sof-%.ri: + ln -sfT intel-signed/sof-$*.ri $@ + + + ################################## + ### Stage sof-tplg/ topologies ### + ################################## + +topologies: + # This requires ./scripts/build-tools.sh -T ... + install ${INSTALL_OPTS} -t ${STAGING_SOF_TPLG}${VERSION_SUFFIX}/ \ + ${BUILDS_ROOT}/tools/build_tools/topology/sof-*.tplg +ifneq (,${VERSION_SUFFIX}) + ln -sfT sof-tplg${VERSION_SUFFIX} ${STAGING_SOF_TPLG} + test -e ${STAGING_SOF_TPLG} +endif + @file ${STAGING_SOF_TPLG} + @tree ${TREE_OPTS} ${STAGING_SOF_TPLG}${VERSION_SUFFIX} | \ + head -n 10; printf '├── ...\n..\n' + + + #################### + ### Self-Testing ### + #################### + +COMPARE_REFS ?= /lib/firmware/intel + +# Useful for testing this Makefile. Can be used against /lib/firmware, +# sof-bin, a previous version of this Makefile,... +# As the first arguments maybe symbolic links, their trailing slash is +# critical. +compare: stage + ! diff -qr --no-dereference ${COMPARE_REFS}/sof/ \ + ${STAGING_SOF_VERSION}/ \ + | grep -v ' differ$$' # || true + ! diff -qr --no-dereference ${COMPARE_REFS}/sof-tplg/ \ + ${STAGING_SOF_TPLG}${VERSION_SUFFIX}/ \ + | grep -v ' differ$$' + +# Invoke this manually to check symbolic links are correct +SIGNED_DUMMIES := ${SIGNED_list:%=${INTEL_SIGNED}/sof-%.ri} \ + ${ALIAS_list:%=${INTEL_SIGNED}/sof-%.ri} +signed_dummies: ${SIGNED_DUMMIES} + ! file $$(find . -type l) | grep -i broken + +${SIGNED_DUMMIES}: | ${INTEL_SIGNED} + touch $@ diff --git a/installer/README.md b/installer/README.md new file mode 100644 index 000000000000..ba1526e9662c --- /dev/null +++ b/installer/README.md @@ -0,0 +1,78 @@ +The GNUmakefile in this directory prepares ``/lib/firmware/intel/sof/`` and +``/lib/firmware/intel/sof-tplg/`` directories. + +It extracts what's needed from the output of the scripts +``./scripts/xtensa-build-all.sh -a`` and ``./scripts/build-tools.sh +-T -l``; running these scripts is a pre-requisite. + +It does not copy anything to ``/lib/firmware/`` directly but to local, +"staging" subdirectory first. The staging area can then be installed with +rsync to a local or remote ``/lib/firmware/intel/`` or to a release +location. This gives an opportunity to inspect the staging area and +avoids running everything as root. + +The default target (re-)generates the staging area: + + make -C installer/ + +Then, to install the staging area: + + sudo make -C installer/ rsync + +By default, the "rsync" target installs to the local +``/lib/firmware/intel/`` directory. To install to a different host or +different directory, copy the ``sample-config.mk`` file to ``config.mk`` +and follow the instructions inside the file. ``config.mk`` can also be +used to change the list of platforms installed and a number of other +Make variables. As usual with Make, many parameters can also be +overridden on the command line. + +To stage and install in one go: + + make -C installer/ stage rsync + +"stage" is the default target and it tries to stage everything: +firmware, dictionaries and topologies. As usual with Make, it's possible +to invoke individual targets. Find a list of targets at the top of +GNUMakefile. + +Sample output: + + staging/sof: symbolic link to sof-v1.6.1 + staging/sof-v1.6.1/ + ├── community/ + │   ├── sof-tgl.ri + │   ├── sof-cnl.ri + │   ├── sof-icl.ri + │   ├── sof-jsl.ri + │   ├── sof-apl.ri + │   ├── sof-cfl.ri -> sof-cnl.ri + │   ├── sof-cml.ri -> sof-cnl.ri + │   ├── sof-ehl.ri -> sof-tgl.ri + │   └── sof-glk.ri -> sof-apl.ri + ├── intel-signed/ + ├── sof-bdw.ri + ├── sof-cht.ri + ├── sof-byt.ri + ├── sof-cnl.ldc + ├── sof-tgl.ldc + ├── sof-icl.ldc + ├── sof-jsl.ldc + ├── sof-apl.ldc + ├── sof-bdw.ldc + ├── sof-byt.ldc + ├── sof-cht.ldc + ├── sof-apl.ri -> intel-signed/sof-apl.ri + ├── sof-cfl.ri -> intel-signed/sof-cfl.ri + ├── sof-cml.ri -> intel-signed/sof-cml.ri + ├── sof-cnl.ri -> intel-signed/sof-cnl.ri + ├── sof-ehl.ri -> intel-signed/sof-ehl.ri + ├── sof-glk.ri -> intel-signed/sof-glk.ri + ├── sof-icl.ri -> intel-signed/sof-icl.ri + ├── sof-jsl.ri -> intel-signed/sof-jsl.ri + ├── sof-tgl.ri -> intel-signed/sof-tgl.ri + ├── sof-cfl.ldc -> sof-cnl.ldc + ├── sof-cml.ldc -> sof-cnl.ldc + ├── sof-ehl.ldc -> sof-tgl.ldc + └── sof-glk.ldc -> sof-apl.ldc + diff --git a/installer/sample-config.mk b/installer/sample-config.mk new file mode 100644 index 000000000000..766704f085ca --- /dev/null +++ b/installer/sample-config.mk @@ -0,0 +1,23 @@ +# To customize the installation, copy this file to config.mk and edit +# it. Leave undefined to use default values. + +# As usual with Make, all these can also be passed as either CLI +# arguments or environment variables. Warning: undefined is NOT the +# same as blank! + +# Everything is installed by default. To install and deploy fewer +# patforms override the default lists like this: +# UNSIGNED_list := +# SIGNED_list := apl tgl + +# The default FW_DESTDIR is the local /lib/firmware/intel directory +# _remote := test-system.local +# FW_DESTDIR := root@${_remote}:/lib/firmware/intel +# USER_DESTDIR := ${_remote}:bin/ + +# Define this empty for a plain sof/ directory and no sof -> sof-v1.2.3 +# symbolic links. +# SOF_VERSION := +# +# SOF_VERSION := $(shell git describe --tags ) +# SOF_VERSION := v1.6.14 From f3a717e6c29e61000c6ea6d4fc500bc70693167b Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 19 Feb 2021 23:58:17 +0000 Subject: [PATCH 04/19] xtensa-build-all.sh: make it runnable from anywhere Fix $pwd confusion and remove bogus ../local/bin PATH that does not exist. Signed-off-by: Marc Herbert (cherry picked from commit e31afb36519fb2d779a1cf4776d203a5f37167f2) --- scripts/xtensa-build-all.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/scripts/xtensa-build-all.sh b/scripts/xtensa-build-all.sh index b07e799b8ae0..ef57fde6e5b4 100755 --- a/scripts/xtensa-build-all.sh +++ b/scripts/xtensa-build-all.sh @@ -14,9 +14,7 @@ BUILD_JOBS=$(nproc --all) BUILD_VERBOSE= PLATFORMS=() -PATH=$pwd/local/bin:$PATH - -pwd=$(pwd) +SOF_TOP=$(cd "$(dirname "$0")/.." && pwd) die() { @@ -136,7 +134,7 @@ then fi OLDPATH=$PATH -WORKDIR="$pwd" +CURDIR="$(pwd)" # build platforms for platform in "${PLATFORMS[@]}" @@ -252,7 +250,7 @@ do # default key for TGL if [ -z "$PRIVATE_KEY_OPTION" ] then - PRIVATE_KEY_OPTION="-D${SIGNING_TOOL}_PRIVATE_KEY=$pwd/keys/otc_private_key_3k.pem" + PRIVATE_KEY_OPTION="-D${SIGNING_TOOL}_PRIVATE_KEY=$SOF_TOP/keys/otc_private_key_3k.pem" fi ;; tgl-h) @@ -265,7 +263,7 @@ do # default key for TGL if [ -z "$PRIVATE_KEY_OPTION" ] then - PRIVATE_KEY_OPTION="-D${SIGNING_TOOL}_PRIVATE_KEY=$pwd/keys/otc_private_key_3k.pem" + PRIVATE_KEY_OPTION="-D${SIGNING_TOOL}_PRIVATE_KEY=$SOF_TOP/keys/otc_private_key_3k.pem" fi ;; jsl) @@ -299,7 +297,7 @@ do ;; esac - ROOT="$pwd/../xtensa-root/$HOST" + ROOT="$SOF_TOP/../xtensa-root/$HOST" if [ -n "$XTENSA_TOOLS_ROOT" ] then @@ -330,7 +328,7 @@ do COMPILER="xcc" else TOOLCHAIN=$HOST - PATH=$pwd/../$HOST/bin:$OLDPATH + PATH=$SOF_TOP/../$HOST/bin:$OLDPATH COMPILER="gcc" case "$platform" in @@ -354,7 +352,7 @@ do -DMEU_OPENSSL="${MEU_OPENSSL}" \ "${MEU_PATH_OPTION}" \ "${PRIVATE_KEY_OPTION}" \ - .. + "$SOF_TOP" cmake --build . -- ${PLATFORM}${DEFCONFIG_PATCH}_defconfig ) @@ -393,7 +391,7 @@ do cmake --build . -- bin -j "${BUILD_JOBS}" ${BUILD_VERBOSE} - cd "$WORKDIR" + cd "$CURDIR" done # for platform in ... # list all the images From 52152f5a5d99597bac2a9b561ea9a7f2446ea80b Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Sat, 20 Feb 2021 01:24:04 +0000 Subject: [PATCH 05/19] build-tools.sh: build in any directory Take $BUILD_TOOLS_DIR from the environment if present. Signed-off-by: Marc Herbert (cherry picked from commit ffaeaaafdd2bc6f89b82b079fdc5229347992b22) --- scripts/build-tools.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/build-tools.sh b/scripts/build-tools.sh index f14cdd74f8bf..6718f915689d 100755 --- a/scripts/build-tools.sh +++ b/scripts/build-tools.sh @@ -24,14 +24,16 @@ EOFUSAGE reconfigure_build() { rm -rf "$BUILD_TOOLS_DIR" - mkdir "$BUILD_TOOLS_DIR" + mkdir -p "$BUILD_TOOLS_DIR" - cd "$BUILD_TOOLS_DIR" - cmake -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" .. + ( cd "$BUILD_TOOLS_DIR" + cmake -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" "${SOF_REPO}/tools" + ) mkdir "$BUILD_TOOLS_DIR/fuzzer" - cd "$BUILD_TOOLS_DIR/fuzzer" - cmake -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" ../../fuzzer + ( cd "$BUILD_TOOLS_DIR/fuzzer" + cmake -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" "${SOF_REPO}/tools/fuzzer" + ) } make_tool() @@ -71,7 +73,7 @@ main() BUILD_ALL SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) SOF_REPO=$(dirname "$SCRIPT_DIR") - BUILD_TOOLS_DIR="$SOF_REPO"/tools/build_tools + : "${BUILD_TOOLS_DIR:=$SOF_REPO/tools/build_tools}" : "${NO_PROCESSORS:=$(nproc)}" BUILD_ALL=false From 6fc6224da308d0c80f0c3b328545a71c9123c28b Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Sat, 20 Feb 2021 01:05:41 +0000 Subject: [PATCH 06/19] installer: new BUILD_TOOLS directory The tools build is independent from the firmware build. The next step is to invoke it from here if needed. Signed-off-by: Marc Herbert (cherry picked from commit f9545c8f59ea3c3c63ed2eb64fa1e6e643a1a46c) --- installer/GNUmakefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/installer/GNUmakefile b/installer/GNUmakefile index 4b7d623c0a16..715e22eeff8c 100644 --- a/installer/GNUmakefile +++ b/installer/GNUmakefile @@ -50,6 +50,7 @@ endif # Our input: build_*_?cc/ directories BUILDS_ROOT ?= ../ +BUILD_TOOLS ?= ${BUILDS_ROOT}/build_tools STAGING_SOF ?= staging/sof STAGING_SOF_VERSION := ${STAGING_SOF}${VERSION_SUFFIX} @@ -85,7 +86,7 @@ ifneq (${USER_DESTDIR},) # TODO: add more user space binaries: sof-ctl, probes,... # absorbe scripts/sof-target-install.sh # Requires ./scripts/build-tools.sh -l ... - rsync -a ${BUILDS_ROOT}/tools/build_tools/logger/sof-logger ${USER_DESTDIR} + rsync -a ${BUILD_TOOLS}/logger/sof-logger ${USER_DESTDIR} endif clean: @@ -166,7 +167,7 @@ ${SIGNED_ALIASES}: ${STAGING_SOF_VERSION}/sof-%.ri: topologies: # This requires ./scripts/build-tools.sh -T ... install ${INSTALL_OPTS} -t ${STAGING_SOF_TPLG}${VERSION_SUFFIX}/ \ - ${BUILDS_ROOT}/tools/build_tools/topology/sof-*.tplg + ${BUILD_TOOLS}/topology/sof-*.tplg ifneq (,${VERSION_SUFFIX}) ln -sfT sof-tplg${VERSION_SUFFIX} ${STAGING_SOF_TPLG} test -e ${STAGING_SOF_TPLG} From 61bee255324ff7416a309f30fd054e87f42022de Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Mon, 22 Feb 2021 06:01:45 +0000 Subject: [PATCH 07/19] installer: (re)build firmware, topologies and user space tools One-touch "make -C installer rsync" combines fast incremental build, staging and deploy in one command. Signed-off-by: Marc Herbert (cherry picked from commit 479809663e2266fdca981dd41607eca79ed0f830) --- installer/GNUmakefile | 45 ++++++++++++++++++++++++++++++++------ installer/sample-config.mk | 7 ++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/installer/GNUmakefile b/installer/GNUmakefile index 715e22eeff8c..dd96b2d4499e 100644 --- a/installer/GNUmakefile +++ b/installer/GNUmakefile @@ -49,7 +49,7 @@ endif ################################ # Our input: build_*_?cc/ directories -BUILDS_ROOT ?= ../ +BUILDS_ROOT ?= ${CURDIR}/../installer-builds BUILD_TOOLS ?= ${BUILDS_ROOT}/build_tools STAGING_SOF ?= staging/sof @@ -67,7 +67,7 @@ endif COMMUNITY := ${STAGING_SOF_VERSION}/community INTEL_SIGNED := ${STAGING_SOF_VERSION}/intel-signed -${COMMUNITY} ${INTEL_SIGNED}: +${COMMUNITY} ${INTEL_SIGNED} ${BUILDS_ROOT}: mkdir -p $@ @@ -85,13 +85,14 @@ rsync: ifneq (${USER_DESTDIR},) # TODO: add more user space binaries: sof-ctl, probes,... # absorbe scripts/sof-target-install.sh - # Requires ./scripts/build-tools.sh -l ... rsync -a ${BUILD_TOOLS}/logger/sof-logger ${USER_DESTDIR} endif clean: ${RM} -r staging/sof* +cleanall: clean + ${RM} -r "${BUILD_TOOLS}/" "${BUILDS_ROOT}"/build_*_?cc/ ########################################################## ### Stage sof-*.ri firmware files and symbolic links #### @@ -124,6 +125,22 @@ ${UNSIGNED_FWS}: ${STAGING_SOF_VERSION}/sof-%.ri: \ install ${INSTALL_OPTS} $< $@ +BUILD_SOF_RIS := \ + ${UNSIGNED_list:%=${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ri} \ + ${SIGNED_list:%=${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ri} + +# The build is not deterministic; use this to reduce noise when testing +# this Makefile +ifneq (true,${BUILD_ONLY_ONCE}) +.PHONY: ${BUILD_SOF_RIS} +endif + +${BUILD_SOF_RIS}: ${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ri: | ${BUILDS_ROOT} + cd ${BUILDS_ROOT} && bdir="$$(dirname $@)" && \ + if test -d $${bdir}; then cmake --build $${bdir} -- sof; else \ + $(CURDIR)/../scripts/xtensa-build-all.sh $*; fi + + ######################################## ### Stage *.ldc logger dictionaries ### ######################################## @@ -134,8 +151,8 @@ LDICTS := ${UNSIGNED_list:%=${STAGING_SOF_VERSION}/sof-%.ldc} \ # $(info LDICTS = ${LDICTS}) ldicts: ${LDICTS} ${LDICTS}: ${STAGING_SOF_VERSION}/sof-%.ldc: \ - ${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ldc - install ${INSTALL_OPTS} $< $@ + ${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ri + install ${INSTALL_OPTS} ${BUILDS_ROOT}/build_$*_${TOOLCHAIN}/sof.ldc $@ ####################################### @@ -164,8 +181,13 @@ ${SIGNED_ALIASES}: ${STAGING_SOF_VERSION}/sof-%.ri: ### Stage sof-tplg/ topologies ### ################################## -topologies: - # This requires ./scripts/build-tools.sh -T ... +# The build is not deterministic; use this to reduce noise when testing +# this Makefile +ifneq (true,${BUILD_ONLY_ONCE}) +.PHONY: ${BUILD_TOOLS} +endif + +topologies: ${BUILD_TOOLS} install ${INSTALL_OPTS} -t ${STAGING_SOF_TPLG}${VERSION_SUFFIX}/ \ ${BUILD_TOOLS}/topology/sof-*.tplg ifneq (,${VERSION_SUFFIX}) @@ -176,6 +198,15 @@ endif @tree ${TREE_OPTS} ${STAGING_SOF_TPLG}${VERSION_SUFFIX} | \ head -n 10; printf '├── ...\n..\n' +# We should use more targets rather than set -e and a multi-lines script +# but that would be verbose. +${BUILD_TOOLS}: + set -e; if test -d ${BUILD_TOOLS}; then \ + for i in topologies sof-logger; do \ + cmake --build ${BUILD_TOOLS} -- $$i; done; else \ + BUILD_TOOLS_DIR=${BUILD_TOOLS} ../scripts/build-tools.sh -T -l ; \ + fi + #################### ### Self-Testing ### diff --git a/installer/sample-config.mk b/installer/sample-config.mk index 766704f085ca..a0aa1730dab6 100644 --- a/installer/sample-config.mk +++ b/installer/sample-config.mk @@ -21,3 +21,10 @@ # # SOF_VERSION := $(shell git describe --tags ) # SOF_VERSION := v1.6.14 + +# Uncomment this to have the build_*_?cc/ directories at the top-level +# BUILDS_ROOT := ${CURDIR}/.. + +# The build is not deterministic; use this to reduce noise when testing +# the installer/ itself +# BUILD_ONLY_ONCE := true From 63c2a641d1bfad1246ffad43ea014a8d97f5d6f6 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Mon, 22 Feb 2021 06:54:00 +0000 Subject: [PATCH 08/19] installer: add checktree target Simple check of the "tree" output Signed-off-by: Marc Herbert (cherry picked from commit 5dea5aead900c2846f72e99409e33f208694fce7) --- installer/GNUmakefile | 12 ++++++++- installer/tests/staging_sof_ref.txt | 38 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 installer/tests/staging_sof_ref.txt diff --git a/installer/GNUmakefile b/installer/GNUmakefile index dd96b2d4499e..ad49f7301b47 100644 --- a/installer/GNUmakefile +++ b/installer/GNUmakefile @@ -90,6 +90,7 @@ endif clean: ${RM} -r staging/sof* + ${RM} ${BUILDS_ROOT}/staging_sof_tree.txt cleanall: clean ${RM} -r "${BUILD_TOOLS}/" "${BUILDS_ROOT}"/build_*_?cc/ @@ -214,7 +215,16 @@ ${BUILD_TOOLS}: COMPARE_REFS ?= /lib/firmware/intel -# Useful for testing this Makefile. Can be used against /lib/firmware, +checktree: + cd ${STAGING_SOF_VERSION} && \ + tree -a --dirsfirst . > ${BUILDS_ROOT}/staging_sof_tree.txt + # Update sof-apl-nocodec.tplg when adding or removing a default platform + diff -u tests/staging_sof_ref.txt ${BUILDS_ROOT}/staging_sof_tree.txt + # Check two random topologies are there + test -f ${STAGING_SOF_TPLG}/sof-apl-nocodec.tplg + test -f ${STAGING_SOF_TPLG}/sof-imx8qxp-nocodec.tplg + +# Useful for testing this Makefile. COMPARE_REFS can be /lib/firmware, # sof-bin, a previous version of this Makefile,... # As the first arguments maybe symbolic links, their trailing slash is # critical. diff --git a/installer/tests/staging_sof_ref.txt b/installer/tests/staging_sof_ref.txt new file mode 100644 index 000000000000..5eef04bfd902 --- /dev/null +++ b/installer/tests/staging_sof_ref.txt @@ -0,0 +1,38 @@ +. +├── community +│   ├── sof-apl.ri +│   ├── sof-cfl.ri -> sof-cnl.ri +│   ├── sof-cml.ri -> sof-cnl.ri +│   ├── sof-cnl.ri +│   ├── sof-ehl.ri -> sof-tgl.ri +│   ├── sof-glk.ri -> sof-apl.ri +│   ├── sof-icl.ri +│   ├── sof-jsl.ri +│   └── sof-tgl.ri +├── intel-signed +├── sof-apl.ldc +├── sof-apl.ri -> intel-signed/sof-apl.ri +├── sof-bdw.ldc +├── sof-bdw.ri +├── sof-byt.ldc +├── sof-byt.ri +├── sof-cfl.ldc -> sof-cnl.ldc +├── sof-cfl.ri -> intel-signed/sof-cfl.ri +├── sof-cht.ldc +├── sof-cht.ri +├── sof-cml.ldc -> sof-cnl.ldc +├── sof-cml.ri -> intel-signed/sof-cml.ri +├── sof-cnl.ldc +├── sof-cnl.ri -> intel-signed/sof-cnl.ri +├── sof-ehl.ldc -> sof-tgl.ldc +├── sof-ehl.ri -> intel-signed/sof-ehl.ri +├── sof-glk.ldc -> sof-apl.ldc +├── sof-glk.ri -> intel-signed/sof-glk.ri +├── sof-icl.ldc +├── sof-icl.ri -> intel-signed/sof-icl.ri +├── sof-jsl.ldc +├── sof-jsl.ri -> intel-signed/sof-jsl.ri +├── sof-tgl.ldc +└── sof-tgl.ri -> intel-signed/sof-tgl.ri + +2 directories, 33 files From 7ecd65b8b69a1b5111d35439c43969e228a22ff0 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 26 Feb 2021 19:19:23 +0000 Subject: [PATCH 09/19] installer/README.md: no need to build manually anymore Doc update missing from the most recent pull request. Also warn against `make -jN stage rsync` Signed-off-by: Marc Herbert (cherry picked from commit 453c686fd12dbd36bda4f2b602167338ee97f6f3) --- installer/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/installer/README.md b/installer/README.md index ba1526e9662c..5819a4a80d89 100644 --- a/installer/README.md +++ b/installer/README.md @@ -2,8 +2,10 @@ The GNUmakefile in this directory prepares ``/lib/firmware/intel/sof/`` and ``/lib/firmware/intel/sof-tplg/`` directories. It extracts what's needed from the output of the scripts -``./scripts/xtensa-build-all.sh -a`` and ``./scripts/build-tools.sh --T -l``; running these scripts is a pre-requisite. +``./scripts/xtensa-build-all.sh`` and ``./scripts/build-tools.sh -T +-l``. It automatically runs these scripts when needed for the platforms +listed in config.mk and performs incremental builds when they have +already been run. It does not copy anything to ``/lib/firmware/`` directly but to local, "staging" subdirectory first. The staging area can then be installed with @@ -36,6 +38,11 @@ firmware, dictionaries and topologies. As usual with Make, it's possible to invoke individual targets. Find a list of targets at the top of GNUMakefile. +You can use `make -jN stage` to build multiple platforms faster but do +*not* `make -jN stage rsync` as this will start deploying before the +builds are all complete. That's because we want the rsync target to be +able to deploy subsets. Instead do: `make -jN somethings && make rsync`. + Sample output: staging/sof: symbolic link to sof-v1.6.1 From ef6d254001cb5ae3790ee3de2bb67d008d5849cb Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Wed, 3 Mar 2021 20:48:49 -0800 Subject: [PATCH 10/19] installer: disable incremental builds with XCC due to $XTENSA_SYSTEM In the future we should probably extract the array of XTENSA_SYSTEM values out of xtensa-build-all.sh Signed-off-by: Marc Herbert (cherry picked from commit 25fc45143b5cca1ac61eede6f19077fd8a5c8e73) --- installer/GNUmakefile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/installer/GNUmakefile b/installer/GNUmakefile index ad49f7301b47..1478191e1a7e 100644 --- a/installer/GNUmakefile +++ b/installer/GNUmakefile @@ -131,14 +131,21 @@ BUILD_SOF_RIS := \ ${SIGNED_list:%=${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ri} # The build is not deterministic; use this to reduce noise when testing -# this Makefile +# this Makefile. +# Also very useful with XCC, see next comment. ifneq (true,${BUILD_ONLY_ONCE}) .PHONY: ${BUILD_SOF_RIS} endif +# Incremental builds are not possible with XCC because the entire +# toolchain requires $XTENSA_SYSTEM but CMake does not support +# build-time environment variables. In the future we could move the +# XTENSA_SYSTEM values out of xtensa-build-all.sh and into some shared +# config file included here. ${BUILD_SOF_RIS}: ${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ri: | ${BUILDS_ROOT} cd ${BUILDS_ROOT} && bdir="$$(dirname $@)" && \ - if test -d $${bdir}; then cmake --build $${bdir} -- sof; else \ + if [ -d $${bdir} ] && [ xcc != "${TOOLCHAIN}" ] ; then \ + cmake --build $${bdir} -- sof; else \ $(CURDIR)/../scripts/xtensa-build-all.sh $*; fi From 3522659f7ec1dfc56d7350a24d18f3c2b006aa61 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 5 Mar 2021 04:02:02 +0000 Subject: [PATCH 11/19] xtensa-buid-all.sh: prepends $OVERRIDE_CONFIG with $SOF_TOP commit e31afb36519f ("xtensa-build-all.sh: make it runnable from anywhere") missed this. Signed-off-by: Marc Herbert (cherry picked from commit 5b34b81cb1d6da846a20dce690b8bf8baa944940) --- scripts/xtensa-build-all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/xtensa-build-all.sh b/scripts/xtensa-build-all.sh index ef57fde6e5b4..30f16a1aa4d3 100755 --- a/scripts/xtensa-build-all.sh +++ b/scripts/xtensa-build-all.sh @@ -83,7 +83,7 @@ SIGNING_TOOL=RIMAGE if [ -n "${OVERRIDE_CONFIG}" ] then - OVERRIDE_CONFIG="src/arch/xtensa/configs/override/$OVERRIDE_CONFIG.config" + OVERRIDE_CONFIG="${SOF_TOP}/src/arch/xtensa/configs/override/$OVERRIDE_CONFIG.config" [ -f "${OVERRIDE_CONFIG}" ] || die 'Invalid override config file %s\n' "${OVERRIDE_CONFIG}" fi @@ -359,7 +359,7 @@ do if [ -n "$OVERRIDE_CONFIG" ] then - cp "../$OVERRIDE_CONFIG" override.config + cp "$OVERRIDE_CONFIG" override.config fi if [[ "x$MAKE_MENUCONFIG" == "xyes" ]] From 163bef9fabce5ea0d3d63e77056d7dde33bc985f Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 26 Feb 2021 06:11:45 +0000 Subject: [PATCH 12/19] xtensa-build-all.sh: show supported XtensaTools installation in usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add this to the -h usage message: This script supports XtensaTools but only when installed in a specific directory structure, example: myXtensa/ └── install/ ├── builds/ │   ├── RD-2012.5-linux/ │   │   └── Intel_HiFiEP/ │   └── RG-2017.8-linux/ │   ├── LX4_langwell_audio_17_8/ │   └── X4H3I16w2D48w3a_2017_8/ └── tools/ ├── RD-2012.5-linux/ │   └── XtensaTools/ └── RG-2017.8-linux/ └── XtensaTools/ $ XTENSA_TOOLS_ROOT=/path/to/myXtensa ./scripts/xtensa-build-all.sh ... Signed-off-by: Marc Herbert (cherry picked from commit 6468490d8d381764adc5ebd259c08a03157986d5) --- scripts/xtensa-build-all.sh | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/xtensa-build-all.sh b/scripts/xtensa-build-all.sh index 30f16a1aa4d3..eb9e75c84e3b 100755 --- a/scripts/xtensa-build-all.sh +++ b/scripts/xtensa-build-all.sh @@ -55,6 +55,25 @@ following environment variable: or: PRIVATE_KEY_OPTION='-DRIMAGE_PRIVATE_KEY=path/to/key' $0 ... +This script supports XtensaTools but only when installed in a specific +directory structure, example: + +myXtensa/ +└── install/ + ├── builds/ + │   ├── RD-2012.5-linux/ + │   │   └── Intel_HiFiEP/ + │   └── RG-2017.8-linux/ + │   ├── LX4_langwell_audio_17_8/ + │   └── X4H3I16w2D48w3a_2017_8/ + └── tools/ + ├── RD-2012.5-linux/ + │   └── XtensaTools/ + └── RG-2017.8-linux/ + └── XtensaTools/ + +$ XTENSA_TOOLS_ROOT=/path/to/myXtensa $0 ... + Supported platforms ${SUPPORTED_PLATFORMS[*]} EOF @@ -297,7 +316,6 @@ do ;; esac - ROOT="$SOF_TOP/../xtensa-root/$HOST" if [ -n "$XTENSA_TOOLS_ROOT" ] then @@ -314,11 +332,15 @@ do XCC="none" XTOBJCOPY="none" XTOBJDUMP="none" - echo "XTENSA_TOOLS_DIR is not a directory" + >&2 printf 'WARNING: %s +\t is not a directory, reverting to gcc\n' "$XTENSA_TOOLS_DIR" fi fi - # update ROOT directory for xt-xcc + # CMake uses ROOT_DIR for includes and libraries a bit like + # --sysroot would. + ROOT="$SOF_TOP/../xtensa-root/$HOST" + if [ "$XCC" == "xt-xcc" ] then TOOLCHAIN=xt From a28c96aade348b47dfce0bebda87a2197e3233ea Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Thu, 4 Mar 2021 09:48:06 -0800 Subject: [PATCH 13/19] installer: TOOLCHAIN default value depends on XTENSA_TOOLS_ROOT XTENSA_TOOLS_ROOT is required by xtensa-build-all.sh anyway, so don't force the user to say twice that they want xcc. Signed-off-by: Marc Herbert (cherry picked from commit 907f8691c2b91e8c66871652342fc6a15d174cee) --- installer/GNUmakefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/installer/GNUmakefile b/installer/GNUmakefile index 1478191e1a7e..678110b740a3 100644 --- a/installer/GNUmakefile +++ b/installer/GNUmakefile @@ -29,7 +29,13 @@ target_of_cml := cnl # Same code, different Intel key target_of_ehl := tgl -TOOLCHAIN ?= gcc +ifeq (,${TOOLCHAIN}) + ifeq (,${XTENSA_TOOLS_ROOT}) + TOOLCHAIN := gcc + else + TOOLCHAIN := xcc + endif +endif TREE_OPTS ?= --sort=size --dirsfirst INSTALL_OPTS ?= -D -p -m 0664 From 654cc1e23e8eb6d4ea1829039c8bdcbf38e3ec69 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Wed, 3 Mar 2021 15:58:13 -0800 Subject: [PATCH 14/19] cmake: print some error messages when XTENSA_TOOLS_ROOT is wrong Notably: xt-xcc --show-config failed with: No such file or directory ... when the directory exists but is wrong. Signed-off-by: Marc Herbert (cherry picked from commit f57de5957cb75051a82846bf0285a03c8d41aac9) --- scripts/cmake/xtensa-toolchain.cmake | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/cmake/xtensa-toolchain.cmake b/scripts/cmake/xtensa-toolchain.cmake index 9c4af027dd01..75414f2b4fe4 100644 --- a/scripts/cmake/xtensa-toolchain.cmake +++ b/scripts/cmake/xtensa-toolchain.cmake @@ -71,12 +71,18 @@ if(XCC) COMMAND ${CMAKE_C_COMPILER} --show-config=config OUTPUT_VARIABLE cc_config_output OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET + RESULT_VARIABLE show_config_res ) + if(NOT ${show_config_res} EQUAL 0) + message(WARNING "${CMAKE_C_COMPILER} --show-config " + "failed with: ${show_config_res}") + endif() string(REGEX MATCH "[a-zA-Z]+-[0-9]+.[0-9]+-[a-zA-Z]*" XCC_TOOLS_VERSION "${cc_config_output}") if(NOT XCC_TOOLS_VERSION) - message(WARNING "Couldn't get compiler description from '${cc_config_output}'") + message(WARNING + "Couldn't get ${CMAKE_C_COMPILER} description," + " --show-config printed: '${cc_config_output}'") set(XCC_TOOLS_VERSION "UNKNOWN-${CMAKE_SYSTEM_NAME}") endif() else() From fdf5b4b7a15f849330216ce10cc5f01e6f2d4eb7 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 5 Mar 2021 02:31:31 +0000 Subject: [PATCH 15/19] installer: add missing mkdir staging/* dep. for symlinks Fixes: rm -rf staging ; make aliases ln: failed to create symbolic link 'staging/sof/sof-glk.ri': No such file or directory This also happens on a brand new checkout when building in parallel with make -j because symbolic links don't have any dependency. Example at: https://github.com/marc-hb/sof/runs/2036288013 Signed-off-by: Marc Herbert (cherry picked from commit 969f370a360a6048b5f676664b9b73d7f9a338a4) --- installer/GNUmakefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/installer/GNUmakefile b/installer/GNUmakefile index 678110b740a3..09ecccc23e0f 100644 --- a/installer/GNUmakefile +++ b/installer/GNUmakefile @@ -73,7 +73,7 @@ endif COMMUNITY := ${STAGING_SOF_VERSION}/community INTEL_SIGNED := ${STAGING_SOF_VERSION}/intel-signed -${COMMUNITY} ${INTEL_SIGNED} ${BUILDS_ROOT}: +${COMMUNITY} ${INTEL_SIGNED} ${BUILDS_ROOT} ${STAGING_SOF_VERSION}: mkdir -p $@ @@ -179,15 +179,15 @@ DICT_ALIASES := ${ALIAS_list:%=${STAGING_SOF_VERSION}/sof-%.ldc} SIGNED_ALIASES := ${ALIAS_list:%=${STAGING_SOF_VERSION}/sof-%.ri} aliases: ${SIGNED_ALIASES} ${COMM_ALIASES} ${DICT_ALIASES} -${COMM_ALIASES}: ${STAGING_SOF_VERSION}/community/sof-%.ri: +${COMM_ALIASES}: ${STAGING_SOF_VERSION}/community/sof-%.ri: | ${COMMUNITY} ln -sfT sof-${target_of_$*}.ri $@ -${DICT_ALIASES}: ${STAGING_SOF_VERSION}/sof-%.ldc: +${DICT_ALIASES}: ${STAGING_SOF_VERSION}/sof-%.ldc: | ${STAGING_SOF_VERSION} ln -sfT sof-${target_of_$*}.ldc $@ # Some have the same key, others just the code. We don't make any # difference here and let the intel-signed/ directory handle this. -${SIGNED_ALIASES}: ${STAGING_SOF_VERSION}/sof-%.ri: +${SIGNED_ALIASES}: ${STAGING_SOF_VERSION}/sof-%.ri: | ${STAGING_SOF_VERSION} ln -sfT intel-signed/sof-$*.ri $@ From c3b6e25f52c7a5632df43d827bb53b9d0637a43a Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Mon, 8 Mar 2021 21:02:47 +0000 Subject: [PATCH 16/19] installer: add tgl-h add tgl-h Signed-off-by: Marc Herbert (cherry picked from commit 1371d479aba900e78bed8090dcf55bd341c56989) --- installer/GNUmakefile | 5 ++++- installer/tests/staging_sof_ref.txt | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/installer/GNUmakefile b/installer/GNUmakefile index 09ecccc23e0f..69db39d8f968 100644 --- a/installer/GNUmakefile +++ b/installer/GNUmakefile @@ -12,9 +12,12 @@ -include config.mk UNSIGNED_list ?= bdw byt cht -SIGNED_list ?= apl cnl icl jsl tgl +SIGNED_list ?= apl cnl icl jsl tgl tgl-h # older SOF versions # SIGNED_list += hsw kbl skl sue + +# In a Linux kernel git clone, try: +# git grep 'sof-.*\.ri' -- sound/soc/ ALIAS_list ?= glk cfl cml ehl $(info UNSIGNED_list = ${UNSIGNED_list} ) diff --git a/installer/tests/staging_sof_ref.txt b/installer/tests/staging_sof_ref.txt index 5eef04bfd902..104fef8dc94b 100644 --- a/installer/tests/staging_sof_ref.txt +++ b/installer/tests/staging_sof_ref.txt @@ -8,6 +8,7 @@ │   ├── sof-glk.ri -> sof-apl.ri │   ├── sof-icl.ri │   ├── sof-jsl.ri +│   ├── sof-tgl-h.ri │   └── sof-tgl.ri ├── intel-signed ├── sof-apl.ldc @@ -32,7 +33,9 @@ ├── sof-icl.ri -> intel-signed/sof-icl.ri ├── sof-jsl.ldc ├── sof-jsl.ri -> intel-signed/sof-jsl.ri +├── sof-tgl-h.ldc +├── sof-tgl-h.ri -> intel-signed/sof-tgl-h.ri ├── sof-tgl.ldc └── sof-tgl.ri -> intel-signed/sof-tgl.ri -2 directories, 33 files +2 directories, 36 files From e58dccbc6f573a4eaa3502a0a730e5b7f5b7f5d9 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Wed, 10 Mar 2021 16:41:31 -0800 Subject: [PATCH 17/19] xtensa-build-all.sh: 3 times more xcc threads than gcc threads As CMake forks one compiler process for each source file, the XTensa compiler spends much more time idle waiting for the license server over the network than actually using CPU or disk. On my VM with 16 virtual cores, rebuilding one platform from scratch with this commit goes down from 12s to less than 9s: more than 25% faster. With Ninja it goes down from 11s to less than 8s. My license server is 25ms away: a closer server does not need as many threads while a more distant server would obviously benefit for even more threads... while already getting an even better improvement than 25% from just 3 times more threads! It's complicated and we probably don't want to start the build by measuring latency to the license server. The entire, purely local _gcc_ build is so fast (~ 1s) that observing any the difference between -j nproc and -j nproc*N is practically impossible so let's not waste RAM when building with gcc. Also: log the $XTENSA_SYSTEM variable as it is required for incremental builds; remove one apostrophe in the here-doc usage as it breaks the parser of some editor (jed). Signed-off-by: Marc Herbert (cherry picked from commit 716efc00bc10fef18d493fb50ee644420275f455) --- scripts/xtensa-build-all.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/xtensa-build-all.sh b/scripts/xtensa-build-all.sh index eb9e75c84e3b..e952460c403d 100755 --- a/scripts/xtensa-build-all.sh +++ b/scripts/xtensa-build-all.sh @@ -16,6 +16,21 @@ PLATFORMS=() SOF_TOP=$(cd "$(dirname "$0")/.." && pwd) +# As CMake forks one compiler process for each source file, the XTensa +# compiler spends much more time idle waiting for the license server +# over the network than actually using CPU or disk. A factor 3 has been +# found optimal for 16 nproc 25ms away from the server; your mileage may +# vary. +# +# The entire, purely local gcc build is so fast (~ 1s) that observing +# any difference between -j nproc and -j nproc*N is practically +# impossible so let's not waste RAM when building with gcc. + +if [ -n "$XTENSA_TOOLS_ROOT" ]; then + BUILD_JOBS=$((BUILD_JOBS * 3)) +fi + + die() { >&2 printf '%s ERROR: ' "$0" @@ -29,7 +44,7 @@ print_usage() { cat <_defconfig file. usage: $0 [options] platform(s) @@ -346,6 +361,7 @@ do TOOLCHAIN=xt ROOT="$XTENSA_BUILDS_DIR/$XTENSA_CORE/xtensa-elf" export XTENSA_SYSTEM=$XTENSA_BUILDS_DIR/$XTENSA_CORE/config + printf 'XTENSA_SYSTEM=%s\n' "${XTENSA_SYSTEM}" PATH=$XTENSA_TOOLS_DIR/XtensaTools/bin:$OLDPATH COMPILER="xcc" else From f3354cc8dd2061784cb5655c0ac7e359b779284b Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Thu, 11 Mar 2021 03:04:51 +0000 Subject: [PATCH 18/19] installer: --no-print-directory Gets the length of a no-op "make topologies" from 380 down to 140 lines. From 300 to 200 for one "make signed" platform. Ninja is more quiet by default. Signed-off-by: Marc Herbert (cherry picked from commit b809caeb5d1b75b5b4472d07fea779572f4ba083) --- installer/GNUmakefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installer/GNUmakefile b/installer/GNUmakefile index 69db39d8f968..858f95960ec6 100644 --- a/installer/GNUmakefile +++ b/installer/GNUmakefile @@ -79,6 +79,9 @@ INTEL_SIGNED := ${STAGING_SOF_VERSION}/intel-signed ${COMMUNITY} ${INTEL_SIGNED} ${BUILDS_ROOT} ${STAGING_SOF_VERSION}: mkdir -p $@ +# The noise for incremental, do-nothing builds is incredible otherwise, +# especially for build_tools +GNUMAKEFLAGS = --no-print-directory ##################################### ### rsync to local or remote #### From 6ed8f21d10f552eafe6c706dfa4e71a218c211fa Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Thu, 11 Mar 2021 02:32:27 +0000 Subject: [PATCH 19/19] installer: fix incremental builds Building "sof" does not rebuild the .ri firmware file. Switch to the "bin" target which is what ./scripts/xtensa-build-all.sh builds. Fixes: 479809663e22 ("installer: (re)build firmware, topologies and user space tools) Signed-off-by: Marc Herbert (cherry picked from commit 21f4e74dd70992e1d5237648b5f2773908dc2f1c) --- installer/GNUmakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/GNUmakefile b/installer/GNUmakefile index 858f95960ec6..2dc89bcc3f27 100644 --- a/installer/GNUmakefile +++ b/installer/GNUmakefile @@ -157,7 +157,7 @@ endif ${BUILD_SOF_RIS}: ${BUILDS_ROOT}/build_%_${TOOLCHAIN}/sof.ri: | ${BUILDS_ROOT} cd ${BUILDS_ROOT} && bdir="$$(dirname $@)" && \ if [ -d $${bdir} ] && [ xcc != "${TOOLCHAIN}" ] ; then \ - cmake --build $${bdir} -- sof; else \ + cmake --build $${bdir} -- bin; else \ $(CURDIR)/../scripts/xtensa-build-all.sh $*; fi