From 4b670c3b3a1afe8aa34855dfe2ffbb6ab7f29a77 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Wed, 3 May 2023 22:38:43 +0000 Subject: [PATCH 1/7] xtensa-build-zephyr.py: parse new versions.json instead of generated .h For rimage parameters, parse new, static versions.json file instead of generated sof_versions.h file. This will make possible to configure rimage before west build. Signed-off-by: Marc Herbert --- scripts/cmake/version.cmake | 1 + scripts/xtensa-build-zephyr.py | 23 +++++++++++------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/cmake/version.cmake b/scripts/cmake/version.cmake index d89106a0380a..54e4d0339493 100644 --- a/scripts/cmake/version.cmake +++ b/scripts/cmake/version.cmake @@ -87,6 +87,7 @@ string(JSON SOF_MICRO ERROR_VARIABLE micro_error if(NOT "${micro_error}" STREQUAL "NOTFOUND") message(STATUS "versions.json: ${micro_error}, defaulting to 0") # TODO: default this to .99 on the main, never released branch like zephyr does + # Keep this default SOF_MICRO the same as the one in xtensa-build-zephyr.py set(SOF_MICRO 0) endif() diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index a5d8c08d6696..9fa60217263e 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -35,6 +35,7 @@ import warnings import fnmatch import hashlib +import json import gzip import dataclasses import concurrent.futures as concurrent @@ -443,11 +444,11 @@ def west_update(): execute_command(["west", "update"], check=True, timeout=3000, cwd=west_top) -def get_sof_version(abs_build_dir): +def get_sof_version(): """[summary] Get version string major.minor.micro of SOF firmware file. When building multiple platforms from the same SOF commit, all platforms share the same version. So for the 1st platform, - generate the version string from sof_version.h and later platforms will + extract the version information from sof/versions.json and later platforms will reuse it. """ global sof_fw_version @@ -455,15 +456,13 @@ def get_sof_version(abs_build_dir): return sof_fw_version versions = {} - with open(pathlib.Path(abs_build_dir, - "zephyr/include/generated/sof_versions.h"), encoding="utf8") as hfile: - for hline in hfile: - words = hline.split() - if words[0] == '#define': - versions[words[1]] = words[2] - sof_fw_version = versions['SOF_MAJOR'] + '.' + versions['SOF_MINOR'] + '.' + \ - versions['SOF_MICRO'] - + with open(SOF_TOP / "versions.json") as versions_file: + versions = json.load(versions_file) + # Keep this default value the same as the default SOF_MICRO in version.cmake + sof_micro = versions['SOF'].get('MICRO', "0") + sof_fw_version = ( + f"{versions['SOF']['MAJOR']}.{versions['SOF']['MINOR']}.{sof_micro}" + ) return sof_fw_version def rmtree_if_exists(directory): @@ -671,7 +670,7 @@ def build_platforms(): sign_cmd += ["--tool-data", str(rimage_config), "--", "-k", str(signing_key)] - sof_fw_vers = get_sof_version(abs_build_dir) + sof_fw_vers = get_sof_version() sign_cmd += ["-f", sof_fw_vers] From 0a18a46c596fa49dd14a0dab77d438e7b27e2f1c Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 12 May 2023 03:18:29 +0000 Subject: [PATCH 2/7] xtensa-build-zephyr.py: add a new line after logging the environment In some cases it was difficult to separate the command's output from the extra environment it ran in. Signed-off-by: Marc Herbert --- scripts/xtensa-build-zephyr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index 9fa60217263e..4cde2648d698 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -277,9 +277,9 @@ def execute_command(*run_args, **run_kwargs): env_arg = run_kwargs.get('env') env_change = set(env_arg.items()) - set(os.environ.items()) if env_arg else None if env_change and (run_kwargs.get('sof_log_env') or args.verbose >= 1): - output += "\n... with extra/modified environment:" + output += "\n... with extra/modified environment:\n" for k_v in env_change: - output += f"\n{k_v[0]}={k_v[1]}" + output += f"{k_v[0]}={k_v[1]}\n" print(output, flush=True) run_kwargs = {k: run_kwargs[k] for k in run_kwargs if not k.startswith("sof_")} From a11fe5e6e20b4a608a3de5cf15dadca7e1e250ee Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Wed, 3 May 2023 23:12:57 +0000 Subject: [PATCH 3/7] xtensa-build-zephyr.py: extract new function rimage_configuration() Zero functional change, pure clean-up. Signed-off-by: Marc Herbert --- scripts/xtensa-build-zephyr.py | 66 +++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index 4cde2648d698..9da03ac6c3e6 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -520,6 +520,43 @@ def build_rimage(): execute_command(rimage_build_cmd, cwd=west_top) +def rimage_configuration(platform_dict): + + sign_cmd = [] + + rimage_executable = shutil.which("rimage", path=RIMAGE_BUILD_DIR) + rimage_config = RIMAGE_SOURCE_DIR / "config" + + sign_cmd += ["--tool-path", rimage_executable] + signing_key = "" + if args.key: + signing_key = args.key + elif "RIMAGE_KEY" in platform_dict: + signing_key = platform_dict["RIMAGE_KEY"] + else: + signing_key = default_rimage_key + + sign_cmd += ["--tool-data", str(rimage_config), "--", "-k", str(signing_key)] + + sof_fw_vers = get_sof_version() + + sign_cmd += ["-f", sof_fw_vers] + + # Default value is 0 in rimage but for Zephyr the "build counter" has always + # been hardcoded to 1 in CMake and there is even a (broken) test that fails + # when it's not hardcoded to 1. + # FIXME: drop this line once the following test is fixed + # tests/avs/fw_00_basic/test_01_load_fw_extended.py::TestLoadFwExtended::():: + # test_00_01_load_fw_and_check_version + sign_cmd += ["-b", "1"] + + if args.ipc == "IPC4": + rimage_desc = pathlib.Path(SOF_TOP, "rimage", "config", platform_dict["IPC4_RIMAGE_DESC"]) + sign_cmd += ["-c", str(rimage_desc)] + + return sign_cmd + + STAGING_DIR = None def build_platforms(): global west_top, SOF_TOP @@ -654,37 +691,10 @@ def build_platforms(): execute_command([str(smex_executable), "-l", str(fw_ldc_file), str(input_elf_file)]) # Sign firmware - rimage_executable = shutil.which("rimage", path=RIMAGE_BUILD_DIR) - rimage_config = RIMAGE_SOURCE_DIR / "config" sign_cmd = ["west"] sign_cmd += ["-v"] * args.verbose sign_cmd += ["sign", "--build-dir", platform_build_dir_name, "--tool", "rimage"] - sign_cmd += ["--tool-path", rimage_executable] - signing_key = "" - if args.key: - signing_key = args.key - elif "RIMAGE_KEY" in platform_dict: - signing_key = platform_dict["RIMAGE_KEY"] - else: - signing_key = default_rimage_key - - sign_cmd += ["--tool-data", str(rimage_config), "--", "-k", str(signing_key)] - - sof_fw_vers = get_sof_version() - - sign_cmd += ["-f", sof_fw_vers] - - # Default value is 0 in rimage but for Zephyr the "build counter" has always - # been hardcoded to 1 in CMake and there is even a (broken) test that fails - # when it's not hardcoded to 1. - # FIXME: drop this line once the following test is fixed - # tests/avs/fw_00_basic/test_01_load_fw_extended.py::TestLoadFwExtended::():: - # test_00_01_load_fw_and_check_version - sign_cmd += ["-b", "1"] - - if args.ipc == "IPC4": - rimage_desc = pathlib.Path(SOF_TOP, "rimage", "config", platform_dict["IPC4_RIMAGE_DESC"]) - sign_cmd += ["-c", str(rimage_desc)] + sign_cmd += rimage_configuration(platform_dict) execute_command(sign_cmd, cwd=west_top) From 86305cc52e533750cbb898a10d43f7ebe2afc533 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Tue, 9 May 2023 22:40:20 +0000 Subject: [PATCH 4/7] xtensa-build-zephyr.py: call rimage_configuration() before building This makes no difference now because it has no side-effect yet but it will make a difference once it starts using `west config`. Signed-off-by: Marc Herbert --- scripts/xtensa-build-zephyr.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index 9da03ac6c3e6..d6aaeb3245d7 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -670,6 +670,11 @@ def build_platforms(): see https://docs.zephyrproject.org/latest/guides/west/build-flash-debug.html#one-time-cmake-arguments Try "west config build.cmake-args -- ..." instead.""") + sign_cmd = ["west"] + sign_cmd += ["-v"] * args.verbose + sign_cmd += ["sign", "--build-dir", platform_build_dir_name, "--tool", "rimage"] + sign_cmd += rimage_configuration(platform_dict) + # Make sure the build logs don't leave anything hidden execute_command(['west', 'config', '-l'], cwd=west_top) @@ -691,11 +696,6 @@ def build_platforms(): execute_command([str(smex_executable), "-l", str(fw_ldc_file), str(input_elf_file)]) # Sign firmware - sign_cmd = ["west"] - sign_cmd += ["-v"] * args.verbose - sign_cmd += ["sign", "--build-dir", platform_build_dir_name, "--tool", "rimage"] - sign_cmd += rimage_configuration(platform_dict) - execute_command(sign_cmd, cwd=west_top) if platform not in RI_INFO_UNSUPPORTED: From ed1d4de16d3e60323da3117e4dc97b8e23381781 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Tue, 9 May 2023 23:36:35 +0000 Subject: [PATCH 5/7] xtensa-build-zephyr.py: stop passing `--tool-data rimage/config/` We used to use `-- -c rimage/config/platf.toml` for the IPC4 overlay and the older `--tool-data rimage/config/` for the rest. This was confusing, inconsistent and used to print this warning: ``` WARNING: --tool-data /var/home/mherber2/SOF/sof/rimage/config ignored, overridden by: -- -c ... ``` `--tool-data` was always a bad idea. It looks generic but it's not; it allows changing only a config directory that is always in the same location. Copy what `west sign` does with `--tool-data`, drop it and use `-- -c` always. Signed-off-by: Marc Herbert --- scripts/xtensa-build-zephyr.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index d6aaeb3245d7..55cc9971f521 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -525,7 +525,6 @@ def rimage_configuration(platform_dict): sign_cmd = [] rimage_executable = shutil.which("rimage", path=RIMAGE_BUILD_DIR) - rimage_config = RIMAGE_SOURCE_DIR / "config" sign_cmd += ["--tool-path", rimage_executable] signing_key = "" @@ -536,7 +535,7 @@ def rimage_configuration(platform_dict): else: signing_key = default_rimage_key - sign_cmd += ["--tool-data", str(rimage_config), "--", "-k", str(signing_key)] + sign_cmd += ["--", "-k", str(signing_key)] sof_fw_vers = get_sof_version() @@ -551,8 +550,11 @@ def rimage_configuration(platform_dict): sign_cmd += ["-b", "1"] if args.ipc == "IPC4": - rimage_desc = pathlib.Path(SOF_TOP, "rimage", "config", platform_dict["IPC4_RIMAGE_DESC"]) - sign_cmd += ["-c", str(rimage_desc)] + rimage_desc = platform_dict["IPC4_RIMAGE_DESC"] + else: + rimage_desc = platform_dict["name"] + ".toml" + + sign_cmd += ["-c", str(RIMAGE_SOURCE_DIR / "config" / rimage_desc)] return sign_cmd From 9960a226736e6c69cab24531047f67421390c7e1 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Wed, 10 May 2023 00:08:18 +0000 Subject: [PATCH 6/7] xtensa-build-zephyr.py: extract new function rimage_options() Unlike rimage_configuration(), rimage_options() will be kept free of side-effect. Signed-off-by: Marc Herbert --- scripts/xtensa-build-zephyr.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index 55cc9971f521..dc19a16a967f 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -526,7 +526,16 @@ def rimage_configuration(platform_dict): rimage_executable = shutil.which("rimage", path=RIMAGE_BUILD_DIR) - sign_cmd += ["--tool-path", rimage_executable] + sign_cmd += ["--tool-path", rimage_executable, "--"] + + sign_cmd += rimage_options(platform_dict) + return sign_cmd + + +def rimage_options(platform_dict): + + sign_cmd = [] + signing_key = "" if args.key: signing_key = args.key @@ -535,7 +544,7 @@ def rimage_configuration(platform_dict): else: signing_key = default_rimage_key - sign_cmd += ["--", "-k", str(signing_key)] + sign_cmd += ["-k", str(signing_key)] sof_fw_vers = get_sof_version() From c907f4a6463815249de02a7e0966e3039506315b Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Fri, 12 May 2023 00:11:35 +0000 Subject: [PATCH 7/7] xtensa-build-zephyr.py: change rimage_options() to return tuples Zero functional change. This will allow excluding some options. Signed-off-by: Marc Herbert --- scripts/xtensa-build-zephyr.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/xtensa-build-zephyr.py b/scripts/xtensa-build-zephyr.py index dc19a16a967f..628d26aa8b04 100755 --- a/scripts/xtensa-build-zephyr.py +++ b/scripts/xtensa-build-zephyr.py @@ -528,13 +528,19 @@ def rimage_configuration(platform_dict): sign_cmd += ["--tool-path", rimage_executable, "--"] - sign_cmd += rimage_options(platform_dict) + # Flatten the list of [ ( "-o", "value" ), ...] tuples + for t in rimage_options(platform_dict): + sign_cmd += t + return sign_cmd def rimage_options(platform_dict): + """Return a list of default rimage options as a list of tuples, + example: [ (-f, 2.5.0), (-b, 1), (-k, key.pem),... ] - sign_cmd = [] + """ + opts = [] signing_key = "" if args.key: @@ -544,11 +550,11 @@ def rimage_options(platform_dict): else: signing_key = default_rimage_key - sign_cmd += ["-k", str(signing_key)] + opts.append(("-k", str(signing_key))) sof_fw_vers = get_sof_version() - sign_cmd += ["-f", sof_fw_vers] + opts.append(("-f", sof_fw_vers)) # Default value is 0 in rimage but for Zephyr the "build counter" has always # been hardcoded to 1 in CMake and there is even a (broken) test that fails @@ -556,16 +562,16 @@ def rimage_options(platform_dict): # FIXME: drop this line once the following test is fixed # tests/avs/fw_00_basic/test_01_load_fw_extended.py::TestLoadFwExtended::():: # test_00_01_load_fw_and_check_version - sign_cmd += ["-b", "1"] + opts.append(("-b", "1")) if args.ipc == "IPC4": rimage_desc = platform_dict["IPC4_RIMAGE_DESC"] else: rimage_desc = platform_dict["name"] + ".toml" - sign_cmd += ["-c", str(RIMAGE_SOURCE_DIR / "config" / rimage_desc)] + opts.append(("-c", str(RIMAGE_SOURCE_DIR / "config" / rimage_desc))) - return sign_cmd + return opts STAGING_DIR = None