Skip to content

xtensa-build-zephyr.py: Deployable build #8643

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lgirdwood merged 4 commits into from
Dec 19, 2023
Merged

xtensa-build-zephyr.py: Deployable build #8643

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
684b4c6
xtensa-build-zephyr.py: Remove IPC4_CONFIG_OVERLAY from PlatformConfig
ujfalusi Dec 15, 2023
8779ab2
xtensa-build-zephyr.py: Fill in missing platform aliases
ujfalusi Dec 18, 2023
6127c71
xtensa-build-zephyr.py: Print out the platform aliases in help
ujfalusi Dec 18, 2023
03abba1
xtensa-build-zephyr.py: Add option to create deployable build
ujfalusi Dec 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 111 additions & 35 deletions scripts/xtensa-build-zephyr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,65 +76,70 @@
# pylint:disable=too-many-instance-attributes
class PlatformConfig:
"Product parameters"
name: str
vendor: str
PLAT_CONFIG: str
XTENSA_TOOLS_VERSION: str
XTENSA_CORE: str
DEFAULT_TOOLCHAIN_VARIANT: str = "xt-clang"
RIMAGE_KEY: pathlib.Path = pathlib.Path(SOF_TOP, "keys", "otc_private_key_3k.pem")
IPC4_CONFIG_OVERLAY: str = "ipc4_overlay.conf"
aliases: list = dataclasses.field(default_factory=list)
ipc4: bool = False

# These can all be built out of the box. --all builds all these.
# Some of these values are duplicated in sof/scripts/set_xtensa_param.sh: keep them in sync.
platform_configs_all = {
# Intel platforms
"tgl" : PlatformConfig(
"tgl", "intel_adsp_cavs25",
"intel", "intel_adsp_cavs25",
f"RG-2017.8{xtensa_tools_version_postfix}",
"cavs2x_LX6HiFi3_2017_8",
"xcc",
aliases = ['adl', 'ehl']
aliases = ['adl', 'adl-n', 'ehl', 'rpl'],
ipc4 = True
),
"tgl-h" : PlatformConfig(
"tgl-h", "intel_adsp_cavs25_tgph",
"intel", "intel_adsp_cavs25_tgph",
f"RG-2017.8{xtensa_tools_version_postfix}",
"cavs2x_LX6HiFi3_2017_8",
"xcc",
aliases = ['adl-s']
aliases = ['adl-s', 'rpl-s'],
ipc4 = True
),
"mtl" : PlatformConfig(
"mtl", "intel_adsp_ace15_mtpm",
"intel", "intel_adsp_ace15_mtpm",
f"RI-2022.10{xtensa_tools_version_postfix}",
"ace10_LX7HiFi4_2022_10",
aliases = ['arl'],
ipc4 = True
),
"lnl" : PlatformConfig(
"lnl", "intel_adsp_ace20_lnl",
"intel", "intel_adsp_ace20_lnl",
f"RI-2022.10{xtensa_tools_version_postfix}",
"ace10_LX7HiFi4_2022_10",
ipc4 = True
),

# NXP platforms
"imx8" : PlatformConfig(
"imx8", "nxp_adsp_imx8",
"imx", "nxp_adsp_imx8",
f"RI-2023.11{xtensa_tools_version_postfix}",
"hifi4_nxp_v5_3_1_prod",
RIMAGE_KEY = "key param ignored by imx8",
),
"imx8x" : PlatformConfig(
"imx8x", "nxp_adsp_imx8x",
"imx", "nxp_adsp_imx8x",
f"RI-2023.11{xtensa_tools_version_postfix}",
"hifi4_nxp_v5_3_1_prod",
RIMAGE_KEY = "key param ignored by imx8x"
),
"imx8m" : PlatformConfig(
"imx8m", "nxp_adsp_imx8m",
"imx", "nxp_adsp_imx8m",
f"RI-2023.11{xtensa_tools_version_postfix}",
"hifi4_mscale_v2_0_2_prod",
RIMAGE_KEY = "key param ignored by imx8m"
),
"imx8ulp" : PlatformConfig(
"imx8ulp", "nxp_adsp_imx8ulp",
"imx", "nxp_adsp_imx8ulp",
f"RI-2023.11{xtensa_tools_version_postfix}",
"hifi4_nxp2_s7_v2_1a_prod",
RIMAGE_KEY = "key param ignored by imx8ulp"
Expand Down Expand Up @@ -178,8 +183,10 @@ def parse_args():
" └── RG-2017.8{}/\n".format(xtensa_tools_version_postfix) +
" └── XtensaTools/\n" +
"$XTENSA_TOOLS_ROOT=/path/to/myXtensa ...\n" +
f"Supported platforms: {list(platform_configs)}"))

f"\nSupported platforms: {list(platform_configs)}"), add_help=False)

parser.add_argument('-h', '--help', action='store_true', help='show help')
parser.add_argument("-a", "--all", required=False, action="store_true",
help="Build all currently supported platforms")
parser.add_argument("platforms", nargs="*", action=validate_platforms_arguments,
Expand Down Expand Up @@ -255,6 +262,28 @@ def parse_args():
help="""Run script in non-interactive mode when user input can not be provided.
This should be used with programmatic script invocations (eg. Continuous Integration).
""")

parser.add_argument("--deployable-build", default=False, action="store_true",
help="""Create a directory structure for the firmware files which can be deployed on target as it is.
This option will cause the --fw-naming and --use-platform-subdir options to be ignored!
The generic, default directory and file structure is IPC version dependent:
IPC3
build-sof-staging/sof/VENDOR/sof/ (on target: /lib/firmware/VENDOR/sof/)
├── community
│ └── sof-PLAT.ri
├── dbgkey
│ └── sof-PLAT.ri
└── sof-PLAT.ri
IPC4
build-sof-staging/sof/VENDOR/sof-ipc4/ (on target: /lib/firmware/VENDOR/sof-ipc4/)
└── PLAT
├── community
│ └── sof-PLAT.ri
├── dbgkey
│ └── sof-PLAT.ri
└── sof-PLAT.ri\n
""")

parser.add_argument("--version", required=False, action="store_true",
help="Prints version of this script.")

Expand All @@ -264,9 +293,22 @@ def parse_args():
args.platforms = list(platform_configs_all)

# print help message if no arguments provided
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
if len(sys.argv) == 1 or args.help:
for platform in platform_configs:
if platform_configs[platform].aliases:
parser.epilog += "\nPlatform aliases for '" + platform + "':\t"
parser.epilog += f"{list(platform_configs[platform].aliases)}"

parser.print_help()
sys.exit(0)

if args.deployable_build:
if args.fw_naming == 'AVS':
args.fw_naming = 'SOF'
print("The option '--fw-naming AVS' is ignored for deployable builds.")
if args.use_platform_subdir:
args.use_platform_subdir = False
print("The option '--use-platform-subdir' is ignored for deployable builds.")

Copy link
Collaborator

@kv2019i kv2019i Dec 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One thing that is not obvious in either git commit or inline comments is what is the relation between deployable_build and the old options. If I've understood right, this is really a transition mechanism, we should not need the old fw-naming/platform-subdir options anymore once. I.e. deployable-build can itself become a deprecate and just be the default. I mean when/why would one need a non-deployable build? :)

Copy link
Contributor Author

@ujfalusi ujfalusi Dec 18, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kv2019i, @marc-hb, we sort of had deployable IPC3 output (no --fw-naming and --use-platform-subdir).
For IPC4 we never had deployable layout, not even with the variation of the two flags.
--fw-naming forces --use-platform-subdir and you will end up with build-sof-staging/sof/tgl/community/dsp_basefw.bin and bunch of .ldc files under sof directory, you still need to sort out the adl, rpl, etc things.
--use-platform-subdir alone would create build-sof-staging/sof/tgl/community/sof-tgl.ri and the alias symlinks right beside the main platform firmware, again, not deployable.

I would argue that the IPC3 was not really deployable either when you mix vendor platforms, they will be dumped at the same directory.

Copy link
Collaborator

@marc-hb marc-hb Dec 19, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would argue that the IPC3 was not really deployable either when you mix vendor platforms, they will be dumped at the same directory.

I "deployed" the IPC3 a lot, it "worked for me". When you write "vendor" do you mean "other than Intel"? Only Intel has ever been released in sof-bin. NXP relies on Yocto AFAIK and ChromeOS has of course a totally different and unique build system. That does not make this script useless outside Intel but I guess it reduces the appeal?

Copy link
Contributor Author

@ujfalusi ujfalusi Dec 19, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@marc-hb,

Supported platforms: ['tgl', 'tgl-h', 'mtl', 'lnl', 'imx8', 'imx8x', 'imx8m', 'imx8ulp']

As long as it claims this, it should be possible to run

./scripts/xtensa-build-zephyr.py --all

and expect the result to be deployable as it is. It is not.

Don't get me wrong, I'm "deploying" the IPC4 firmware to my DUTs just fine:

scp ../build-sof-staging/sof/tgl/community/dsp_basefw.bin root@<tgl_DUT>:/lib/firmware/intel/sof-ipc4/tgl/community/sof-tgl.ri

but I cannot just grab the build dir and move it to tgz or to DUT as it is.

if args.fw_naming == 'AVS':
if not args.use_platform_subdir:
Expand Down Expand Up @@ -304,18 +346,18 @@ def execute_command(*run_args, **run_kwargs):

return subprocess.run(*run_args, **run_kwargs)

def symlink_or_copy(directory, origbase, newbase):
"""Create a symbolic link or copy in the same directory. Don't
bother Windows users with symbolic links because they require
special privileges. Windows don't care about /lib/firmware/sof/
anyway. Make a copy instead to preserve cross-platform consistency.

def symlink_or_copy(targetdir, targetfile, linkdir, linkfile):
"""Create a relative symbolic link or copy. Don't bother Windows users
with symbolic links because they require special privileges.
Windows don't care about /lib/firmware/sof/ anyway.
Make a copy instead to preserve cross-platform consistency.
"""
new = pathlib.Path(directory) / newbase
target = pathlib.Path(targetdir) / targetfile
link = pathlib.Path(linkdir) / linkfile
if py_platform.system() == "Windows":
shutil.copy2(pathlib.Path(directory) / origbase, new)
shutil.copy2(target, link)
else:
new.symlink_to(origbase)
link.symlink_to(os.path.relpath(target, linkdir))

def show_installed_files():
"""[summary] Scans output directory building binary tree from files and folders
Expand Down Expand Up @@ -513,6 +555,12 @@ def clean_staging(platform):
for f in sof_output_dir.glob(f"**/sof-{p}.*"):
os.remove(f)

# remove IPC4 deployable build directories
if platform_configs[platform].ipc4:
sof_platform_output_dir = sof_output_dir / platform_configs[platform].vendor / "sof-ipc4"
rmtree_if_exists(sof_platform_output_dir / platform)
for p_alias in platform_configs[platform].aliases:
rmtree_if_exists(sof_platform_output_dir / p_alias)

RIMAGE_BUILD_DIR = west_top / "build-rimage"

Expand Down Expand Up @@ -656,11 +704,21 @@ def build_platforms():
sof_output_dir.mkdir(parents=True, exist_ok=True)
for platform in args.platforms:
platf_build_environ = os.environ.copy()
if args.use_platform_subdir:
sof_platform_output_dir = pathlib.Path(sof_output_dir, platform)

if args.deployable_build:
vendor_output_dir = pathlib.Path(sof_output_dir, platform_configs[platform].vendor)
if platform_configs[platform].ipc4:
sof_platform_output_dir = pathlib.Path(vendor_output_dir, "sof-ipc4")
else:
sof_platform_output_dir = pathlib.Path(vendor_output_dir, "sof")

sof_platform_output_dir.mkdir(parents=True, exist_ok=True)
else:
sof_platform_output_dir = sof_output_dir
if args.use_platform_subdir:
sof_platform_output_dir = pathlib.Path(sof_output_dir, platform)
sof_platform_output_dir.mkdir(parents=True, exist_ok=True)
else:
sof_platform_output_dir = sof_output_dir

# For now convert the new dataclass to what it used to be
_dict = dataclasses.asdict(platform_configs[platform])
Expand Down Expand Up @@ -763,7 +821,7 @@ def build_platforms():
execute_command([str(smex_executable), "-l", str(fw_ldc_file), str(input_elf_file)])

for p_alias in platform_configs[platform].aliases:
symlink_or_copy(sof_platform_output_dir, f"sof-{platform}.ldc", f"sof-{p_alias}.ldc")
symlink_or_copy(sof_platform_output_dir, f"sof-{platform}.ldc", sof_platform_output_dir, f"sof-{p_alias}.ldc")

# reproducible-zephyr.ri is less useful now that show_installed_files() shows
# checksums too. However: - it's still useful when only the .ri file is
Expand Down Expand Up @@ -805,7 +863,7 @@ def build_platforms():
symlinks=True, ignore_dangling_symlinks=True, dirs_exist_ok=True)


def install_platform(platform, sof_platform_output_dir, platf_build_environ):
def install_platform(platform, sof_output_dir, platf_build_environ):

# Keep in sync with caller
platform_build_dir_name = f"build-{platform}"
Expand All @@ -817,21 +875,39 @@ def install_platform(platform, sof_platform_output_dir, platf_build_environ):
# Disguise ourselves for local testing purposes
output_fwname = "dsp_basefw.bin"
else:
# Regular name
# Regular name (deployable build also uses SOF naming convention)
output_fwname = "".join(["sof-", platform, ".ri"])

install_key_dir = sof_platform_output_dir
if args.deployable_build and platform_configs[platform].ipc4:
install_key_dir = pathlib.Path(sof_output_dir, platform)
else:
install_key_dir = sof_output_dir

if args.key_type_subdir != "none":
install_key_dir = install_key_dir / args.key_type_subdir

os.makedirs(install_key_dir, exist_ok=True)
# looses file owner and group - file is commonly accessible
shutil.copy2(abs_build_dir / "zephyr.ri", install_key_dir / output_fwname)

# The production key is usually different
if args.key_type_subdir != "none" and args.fw_naming != "AVS":
if args.deployable_build and platform_configs[platform].ipc4:
# IPC4 deployable builds are using separate directories per platforms
# create the structure and symlinks for the alias platforms
for p_alias in platform_configs[platform].aliases:
symlink_or_copy(install_key_dir, output_fwname, f"sof-{p_alias}.ri")
alias_fwname = "".join(["sof-", p_alias, ".ri"])

alias_key_dir = pathlib.Path(sof_output_dir, p_alias)
if args.key_type_subdir != "none":
alias_key_dir = alias_key_dir / args.key_type_subdir

os.makedirs(alias_key_dir, exist_ok=True)
symlink_or_copy(install_key_dir, output_fwname, alias_key_dir, alias_fwname)
else:
# non deployable builds and IPC3 deployable builds are using the same symlink scheme
# The production key is usually different
if args.key_type_subdir != "none" and args.fw_naming != "AVS":
for p_alias in platform_configs[platform].aliases:
symlink_or_copy(install_key_dir, output_fwname, install_key_dir, f"sof-{p_alias}.ri")


# sof-info/ directory
Expand Down