[TEST IGNORE] Big sign switch

scripts/cmake/version.cmake

@@ -87,6 +87,7 @@ string(JSON SOF_MICRO ERROR_VARIABLE micro_error
 if(NOT "${micro_error}" STREQUAL "NOTFOUND")
 	message(STATUS "versions.json: ${micro_error}, defaulting to 0")
 	# TODO: default this to .99 on the main, never released branch like zephyr does
+	# Keep this default SOF_MICRO the same as the one in xtensa-build-zephyr.py
 	set(SOF_MICRO 0)
 endif()
 

scripts/xtensa-build-zephyr.py

@@ -35,10 +35,13 @@
 import warnings
 import fnmatch
 import hashlib
+import json
 import gzip
 import dataclasses
 import concurrent.futures as concurrent
 
+from west import configuration as west_config
+
 # anytree module is defined in Zephyr build requirements
 from anytree import AnyNode, RenderTree, render
 from packaging import version
@@ -276,9 +279,9 @@ def execute_command(*run_args, **run_kwargs):
 		env_arg = run_kwargs.get('env')
 		env_change = set(env_arg.items()) - set(os.environ.items()) if env_arg else None
 		if env_change and (run_kwargs.get('sof_log_env') or args.verbose >= 1):
-			output += "\n... with extra/modified environment:"
+			output += "\n... with extra/modified environment:\n"
 			for k_v in env_change:
-				output += f"\n{k_v[0]}={k_v[1]}"
+				output += f"{k_v[0]}={k_v[1]}\n"
 		print(output, flush=True)
 
 	run_kwargs = {k: run_kwargs[k] for k in run_kwargs if not k.startswith("sof_")}
@@ -376,6 +379,8 @@ def west_reinitialize(west_root_dir: pathlib.Path, west_manifest_path: pathlib.P
 	shutil.rmtree(dot_west_directory)
 	execute_command(["west", "init", "-l", f"{SOF_TOP}"], cwd=west_top)
 
+
+# TODO: use the west_config API instead of execute_command()
 def west_init_if_needed():
 	"""[summary] Validates whether west workspace had been initialized and points to SOF manifest.
 	Peforms west initialization if needed.
@@ -443,27 +448,26 @@ def west_update():
 	execute_command(["west", "update"], check=True, timeout=3000, cwd=west_top)
 
 
-def get_sof_version(abs_build_dir):
+def get_sof_version():
 	"""[summary] Get version string major.minor.micro of SOF
 	firmware file. When building multiple platforms from the same SOF
 	commit, all platforms share the same version. So for the 1st platform,
-	generate the version string from sof_version.h and later platforms will
+	extract the version information from sof/versions.json and later platforms will
 	reuse it.
 	"""
 	global sof_fw_version
 	if sof_fw_version:
 		return sof_fw_version
 
 	versions = {}
-	with open(pathlib.Path(abs_build_dir,
-		  "zephyr/include/generated/sof_versions.h"), encoding="utf8") as hfile:
-		for hline in hfile:
-			words = hline.split()
-			if words[0] == '#define':
-				versions[words[1]] = words[2]
-	sof_fw_version = versions['SOF_MAJOR'] + '.' + versions['SOF_MINOR'] + '.' + \
-		      versions['SOF_MICRO']
-
+	with open(SOF_TOP / "versions.json") as versions_file:
+		versions = json.load(versions_file)
+		sof_micro = versions['SOF'].get('MICRO')
+		# Keep this default the same as SOF_MICRO in version.cmake
+		sof_micro = '0' if sof_micro is None else sof_micro
+		sof_fw_version = (
+			f"{versions['SOF']['MAJOR']}.{versions['SOF']['MINOR']}.{sof_micro}"
+		)
 	return sof_fw_version
 
 def rmtree_if_exists(directory):
@@ -496,6 +500,53 @@ def clean_staging(platform):
 # detailed comments in west.yml
 RIMAGE_SOURCE_DIR = west_top / "sof" / "rimage"
 
+
+def rimage_west_configuration(platform_dict, dest_dir):
+	"""Configure rimage in a new file `dest_dir/west.conf`, starting
+	from the workspace .west/config.
+	Returns the pathlib.Path to the new file.
+	"""
+
+	saved_local_var = os.environ.get('WEST_CONFIG_LOCAL')
+	workspace_west_config_path = os.environ.get('WEST_CONFIG_LOCAL',
+						   str(west_top / ".west" / "config"))
+	platform_west_config_path = dest_dir / "westconfig.ini"
+	dest_dir.mkdir(parents=True, exist_ok=True)
+	shutil.copyfile(workspace_west_config_path, platform_west_config_path)
+
+	# Create `platform_wconfig` object pointing at our copy
+	os.environ['WEST_CONFIG_LOCAL'] = str(platform_west_config_path)
+	platform_wconfig = west_config.Configuration()
+	if saved_local_var is None:
+		del os.environ['WEST_CONFIG_LOCAL']
+	else:
+		os.environ['WEST_CONFIG_LOCAL'] = saved_local_var
+
+	# By default, run rimage directly from the rimage build directory
+	if platform_wconfig.get("rimage.path") is None:
+		rimage_executable = shutil.which("rimage", path=RIMAGE_BUILD_DIR)
+		assert pathlib.Path(str(rimage_executable)).exists()
+		platform_wconfig.set("rimage.path", shlex.quote(rimage_executable),
+				     west_config.ConfigFile.LOCAL)
+
+	_ws_args = platform_wconfig.get("rimage.extra-args")
+	workspace_extra_args = [] if _ws_args is None else shlex.split(_ws_args)
+
+	# Flatten default rimage options while giving precedence to the workspace =
+	# the user input. We could just append and leave duplicates but that would be
+	# at best confusing and at worst relying on undocumented rimage precedence.
+	extra_args = []
+	for default_opt in rimage_options(platform_dict):
+		if not default_opt[0] in workspace_extra_args:
+			extra_args += default_opt
+
+	extra_args += workspace_extra_args
+
+	platform_wconfig.set("rimage.extra-args", shlex.join(extra_args))
+
+	return platform_west_config_path
+
+
 def build_rimage():
 
 	# Detect non-west rimage duplicates, example: git submdule
@@ -521,6 +572,48 @@ def build_rimage():
 	execute_command(rimage_build_cmd, cwd=west_top)
 
 
+def rimage_options(platform_dict):
+	"""Return a list of default rimage options as a list of tuples,
+	example: [ (-f, 2.5.0), (-b, 1), (-k, key.pem),... ]
+
+	"""
+	opts = []
+
+	if args.verbose > 0:
+		opts.append(("-v",) * args.verbose)
+
+	signing_key = ""
+	if args.key:
+		signing_key = args.key
+	elif "RIMAGE_KEY" in platform_dict:
+		signing_key = platform_dict["RIMAGE_KEY"]
+	else:
+		signing_key = default_rimage_key
+
+	opts.append(("-k", str(signing_key)))
+
+	sof_fw_vers = get_sof_version()
+
+	opts.append(("-f", sof_fw_vers))
+
+	# Default value is 0 in rimage but for Zephyr the "build counter" has always
+	# been hardcoded to 1 in CMake and there is even a (broken) test that fails
+	# when it's not hardcoded to 1.
+	# FIXME: drop this line once the following test is fixed
+	# tests/avs/fw_00_basic/test_01_load_fw_extended.py::TestLoadFwExtended::()::
+	#                         test_00_01_load_fw_and_check_version
+	opts.append(("-b", "1"))
+
+	if args.ipc == "IPC4":
+		rimage_desc = platform_dict["IPC4_RIMAGE_DESC"]
+	else:
+		rimage_desc = platform_dict["name"] + ".toml"
+
+	opts.append(("-c", str(RIMAGE_SOURCE_DIR / "config" / rimage_desc)))
+
+	return opts
+
+
 STAGING_DIR = None
 def build_platforms():
 	global west_top, SOF_TOP
@@ -634,8 +727,15 @@ def build_platforms():
 	see https://docs.zephyrproject.org/latest/guides/west/build-flash-debug.html#one-time-cmake-arguments
 	Try "west config build.cmake-args -- ..." instead.""")
 
+		platf_build_environ['WEST_CONFIG_LOCAL'] = str(rimage_west_configuration(
+			platform_dict,
+			STAGING_DIR / "sof-info" / platform
+		))
+
 		# Make sure the build logs don't leave anything hidden
-		execute_command(['west', 'config', '-l'], cwd=west_top)
+		execute_command(['west', 'config', '-l'], cwd=west_top,
+				env=platf_build_environ, sof_log_env=True)
+		print()
 
 		# Build
 		try:
@@ -654,41 +754,6 @@ def build_platforms():
 		# Extract metadata
 		execute_command([str(smex_executable), "-l", str(fw_ldc_file), str(input_elf_file)])
 
-		# Sign firmware
-		rimage_executable = shutil.which("rimage", path=RIMAGE_BUILD_DIR)
-		rimage_config = RIMAGE_SOURCE_DIR / "config"
-		sign_cmd = ["west"]
-		sign_cmd += ["-v"] * args.verbose
-		sign_cmd += ["sign", "--build-dir", platform_build_dir_name, "--tool", "rimage"]
-		sign_cmd += ["--tool-path", rimage_executable]
-		signing_key = ""
-		if args.key:
-			signing_key = args.key
-		elif "RIMAGE_KEY" in platform_dict:
-			signing_key = platform_dict["RIMAGE_KEY"]
-		else:
-			signing_key = default_rimage_key
-
-		sign_cmd += ["--tool-data", str(rimage_config), "--", "-k", str(signing_key)]
-
-		sof_fw_vers = get_sof_version(abs_build_dir)
-
-		sign_cmd += ["-f", sof_fw_vers]
-
-		# Default value is 0 in rimage but for Zephyr the "build counter" has always
-		# been hardcoded to 1 in CMake and there is even a (broken) test that fails
-		# when it's not hardcoded to 1.
-		# FIXME: drop this line once the following test is fixed
-		# tests/avs/fw_00_basic/test_01_load_fw_extended.py::TestLoadFwExtended::()::
-		#                       test_00_01_load_fw_and_check_version
-		sign_cmd += ["-b", "1"]
-
-		if args.ipc == "IPC4":
-			rimage_desc = pathlib.Path(SOF_TOP, "rimage", "config", platform_dict["IPC4_RIMAGE_DESC"])
-			sign_cmd += ["-c", str(rimage_desc)]
-
-		execute_command(sign_cmd, cwd=west_top)
-
 		if platform not in RI_INFO_UNSUPPORTED:
 			reproducible_checksum(platform, west_top / platform_build_dir_name / "zephyr" / "zephyr.ri")
 

west.yml

@@ -43,7 +43,7 @@ manifest:
 
     - name: zephyr
       repo-path: zephyr
-      revision: 9fc99928caf04c3ae78871f803ce8cee83541ae0
+      revision: 6697c5aa0bb296ad008ef8f728df254954764962
       remote: zephyrproject
 
       # Import some projects listed in zephyr/west.yml@revision

zephyr/scripts/clean-expected-release-differences.sh

@@ -19,7 +19,7 @@ fix_dir()
         die 'No %s/build-sof-staging directory\n' "$bd"
 
     # config files have absolute paths
-    find "$bd" -name 'config.gz' -exec rm '{}' \;
+    find "$bd" \( -name 'config.gz' -o -name 'westconfig.ini' \) -exec rm '{}' \;
 
     # In case of a compression timestamp. Also gives better messages.
     find "$bd" -name '*.gz' -print0 | xargs -r -0 gunzip