diff --git a/scripts/cmake/version.cmake b/scripts/cmake/version.cmake index d89106a0380a..54e4d0339493 100644 --- a/scripts/cmake/version.cmake +++ b/scripts/cmake/version.cmake @@ -87,6 +87,7 @@ string(JSON SOF_MICRO ERROR_VARIABLE micro_error if(NOT "${micro_error}" STREQUAL "NOTFOUND") message(STATUS "versions.json: ${micro_error}, defaulting to 0") # TODO: default this to .99 on the main, never released branch like zephyr does + # Keep this default SOF_MICRO the same as the one in xtensa-build-zephyr.py set(SOF_MICRO 0) endif() diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index a5d8c08d6696..3897a0a676a4 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -35,10 +35,13 @@ import warnings import fnmatch import hashlib +import json import gzip import dataclasses import concurrent.futures as concurrent +from west import configuration as west_config + # anytree module is defined in Zephyr build requirements from anytree import AnyNode, RenderTree, render from packaging import version @@ -276,9 +279,9 @@ def execute_command(*run_args, **run_kwargs): env_arg = run_kwargs.get('env') env_change = set(env_arg.items()) - set(os.environ.items()) if env_arg else None if env_change and (run_kwargs.get('sof_log_env') or args.verbose >= 1): - output += "\n... with extra/modified environment:" + output += "\n... with extra/modified environment:\n" for k_v in env_change: - output += f"\n{k_v[0]}={k_v[1]}" + output += f"{k_v[0]}={k_v[1]}\n" print(output, flush=True) run_kwargs = {k: run_kwargs[k] for k in run_kwargs if not k.startswith("sof_")} @@ -376,6 +379,8 @@ def west_reinitialize(west_root_dir: pathlib.Path, west_manifest_path: pathlib.P shutil.rmtree(dot_west_directory) execute_command(["west", "init", "-l", f"{SOF_TOP}"], cwd=west_top) + +# TODO: use the west_config API instead of execute_command() def west_init_if_needed(): """[summary] Validates whether west workspace had been initialized and points to SOF manifest. Peforms west initialization if needed. @@ -443,11 +448,11 @@ def west_update(): execute_command(["west", "update"], check=True, timeout=3000, cwd=west_top) -def get_sof_version(abs_build_dir): +def get_sof_version(): """[summary] Get version string major.minor.micro of SOF firmware file. When building multiple platforms from the same SOF commit, all platforms share the same version. So for the 1st platform, - generate the version string from sof_version.h and later platforms will + extract the version information from sof/versions.json and later platforms will reuse it. """ global sof_fw_version @@ -455,15 +460,14 @@ def get_sof_version(abs_build_dir): return sof_fw_version versions = {} - with open(pathlib.Path(abs_build_dir, - "zephyr/include/generated/sof_versions.h"), encoding="utf8") as hfile: - for hline in hfile: - words = hline.split() - if words[0] == '#define': - versions[words[1]] = words[2] - sof_fw_version = versions['SOF_MAJOR'] + '.' + versions['SOF_MINOR'] + '.' + \ - versions['SOF_MICRO'] - + with open(SOF_TOP / "versions.json") as versions_file: + versions = json.load(versions_file) + sof_micro = versions['SOF'].get('MICRO') + # Keep this default the same as SOF_MICRO in version.cmake + sof_micro = '0' if sof_micro is None else sof_micro + sof_fw_version = ( + f"{versions['SOF']['MAJOR']}.{versions['SOF']['MINOR']}.{sof_micro}" + ) return sof_fw_version def rmtree_if_exists(directory): @@ -496,6 +500,53 @@ def clean_staging(platform): # detailed comments in west.yml RIMAGE_SOURCE_DIR = west_top / "sof" / "rimage" + +def rimage_west_configuration(platform_dict, dest_dir): + """Configure rimage in a new file `dest_dir/west.conf`, starting + from the workspace .west/config. + Returns the pathlib.Path to the new file. + """ + + saved_local_var = os.environ.get('WEST_CONFIG_LOCAL') + workspace_west_config_path = os.environ.get('WEST_CONFIG_LOCAL', + str(west_top / ".west" / "config")) + platform_west_config_path = dest_dir / "westconfig.ini" + dest_dir.mkdir(parents=True, exist_ok=True) + shutil.copyfile(workspace_west_config_path, platform_west_config_path) + + # Create `platform_wconfig` object pointing at our copy + os.environ['WEST_CONFIG_LOCAL'] = str(platform_west_config_path) + platform_wconfig = west_config.Configuration() + if saved_local_var is None: + del os.environ['WEST_CONFIG_LOCAL'] + else: + os.environ['WEST_CONFIG_LOCAL'] = saved_local_var + + # By default, run rimage directly from the rimage build directory + if platform_wconfig.get("rimage.path") is None: + rimage_executable = shutil.which("rimage", path=RIMAGE_BUILD_DIR) + assert pathlib.Path(str(rimage_executable)).exists() + platform_wconfig.set("rimage.path", shlex.quote(rimage_executable), + west_config.ConfigFile.LOCAL) + + _ws_args = platform_wconfig.get("rimage.extra-args") + workspace_extra_args = [] if _ws_args is None else shlex.split(_ws_args) + + # Flatten default rimage options while giving precedence to the workspace = + # the user input. We could just append and leave duplicates but that would be + # at best confusing and at worst relying on undocumented rimage precedence. + extra_args = [] + for default_opt in rimage_options(platform_dict): + if not default_opt[0] in workspace_extra_args: + extra_args += default_opt + + extra_args += workspace_extra_args + + platform_wconfig.set("rimage.extra-args", shlex.join(extra_args)) + + return platform_west_config_path + + def build_rimage(): # Detect non-west rimage duplicates, example: git submdule @@ -521,6 +572,48 @@ def build_rimage(): execute_command(rimage_build_cmd, cwd=west_top) +def rimage_options(platform_dict): + """Return a list of default rimage options as a list of tuples, + example: [ (-f, 2.5.0), (-b, 1), (-k, key.pem),... ] + + """ + opts = [] + + if args.verbose > 0: + opts.append(("-v",) * args.verbose) + + signing_key = "" + if args.key: + signing_key = args.key + elif "RIMAGE_KEY" in platform_dict: + signing_key = platform_dict["RIMAGE_KEY"] + else: + signing_key = default_rimage_key + + opts.append(("-k", str(signing_key))) + + sof_fw_vers = get_sof_version() + + opts.append(("-f", sof_fw_vers)) + + # Default value is 0 in rimage but for Zephyr the "build counter" has always + # been hardcoded to 1 in CMake and there is even a (broken) test that fails + # when it's not hardcoded to 1. + # FIXME: drop this line once the following test is fixed + # tests/avs/fw_00_basic/test_01_load_fw_extended.py::TestLoadFwExtended::():: + # test_00_01_load_fw_and_check_version + opts.append(("-b", "1")) + + if args.ipc == "IPC4": + rimage_desc = platform_dict["IPC4_RIMAGE_DESC"] + else: + rimage_desc = platform_dict["name"] + ".toml" + + opts.append(("-c", str(RIMAGE_SOURCE_DIR / "config" / rimage_desc))) + + return opts + + STAGING_DIR = None def build_platforms(): global west_top, SOF_TOP @@ -634,8 +727,15 @@ def build_platforms(): see https://docs.zephyrproject.org/latest/guides/west/build-flash-debug.html#one-time-cmake-arguments Try "west config build.cmake-args -- ..." instead.""") + platf_build_environ['WEST_CONFIG_LOCAL'] = str(rimage_west_configuration( + platform_dict, + STAGING_DIR / "sof-info" / platform + )) + # Make sure the build logs don't leave anything hidden - execute_command(['west', 'config', '-l'], cwd=west_top) + execute_command(['west', 'config', '-l'], cwd=west_top, + env=platf_build_environ, sof_log_env=True) + print() # Build try: @@ -654,41 +754,6 @@ def build_platforms(): # Extract metadata execute_command([str(smex_executable), "-l", str(fw_ldc_file), str(input_elf_file)]) - # Sign firmware - rimage_executable = shutil.which("rimage", path=RIMAGE_BUILD_DIR) - rimage_config = RIMAGE_SOURCE_DIR / "config" - sign_cmd = ["west"] - sign_cmd += ["-v"] * args.verbose - sign_cmd += ["sign", "--build-dir", platform_build_dir_name, "--tool", "rimage"] - sign_cmd += ["--tool-path", rimage_executable] - signing_key = "" - if args.key: - signing_key = args.key - elif "RIMAGE_KEY" in platform_dict: - signing_key = platform_dict["RIMAGE_KEY"] - else: - signing_key = default_rimage_key - - sign_cmd += ["--tool-data", str(rimage_config), "--", "-k", str(signing_key)] - - sof_fw_vers = get_sof_version(abs_build_dir) - - sign_cmd += ["-f", sof_fw_vers] - - # Default value is 0 in rimage but for Zephyr the "build counter" has always - # been hardcoded to 1 in CMake and there is even a (broken) test that fails - # when it's not hardcoded to 1. - # FIXME: drop this line once the following test is fixed - # tests/avs/fw_00_basic/test_01_load_fw_extended.py::TestLoadFwExtended::():: - # test_00_01_load_fw_and_check_version - sign_cmd += ["-b", "1"] - - if args.ipc == "IPC4": - rimage_desc = pathlib.Path(SOF_TOP, "rimage", "config", platform_dict["IPC4_RIMAGE_DESC"]) - sign_cmd += ["-c", str(rimage_desc)] - - execute_command(sign_cmd, cwd=west_top) - if platform not in RI_INFO_UNSUPPORTED: reproducible_checksum(platform, west_top / platform_build_dir_name / "zephyr" / "zephyr.ri") diff --git a/west.yml b/west.yml index d24e69e96c8f..ef0d80765b3a 100644 --- a/west.yml +++ b/west.yml @@ -43,7 +43,7 @@ manifest: - name: zephyr repo-path: zephyr - revision: 9fc99928caf04c3ae78871f803ce8cee83541ae0 + revision: 6697c5aa0bb296ad008ef8f728df254954764962 remote: zephyrproject # Import some projects listed in zephyr/west.yml@revision diff --git a/zephyr/scripts/clean-expected-release-differences.sh b/zephyr/scripts/clean-expected-release-differences.sh index 7a1f0af2aac4..b04c61bb8255 100755 --- a/zephyr/scripts/clean-expected-release-differences.sh +++ b/zephyr/scripts/clean-expected-release-differences.sh @@ -19,7 +19,7 @@ fix_dir() die 'No %s/build-sof-staging directory\n' "$bd" # config files have absolute paths - find "$bd" -name 'config.gz' -exec rm '{}' \; + find "$bd" \( -name 'config.gz' -o -name 'westconfig.ini' \) -exec rm '{}' \; # In case of a compression timestamp. Also gives better messages. find "$bd" -name '*.gz' -print0 | xargs -r -0 gunzip