From 43109456c1017cf72d136f9ea51264510e800ab9 Mon Sep 17 00:00:00 2001 From: Justin Tracey Date: Fri, 31 Oct 2025 20:37:12 -0400 Subject: [PATCH 1/3] feat: add thunderbird support --- src/fuzzfetch/args.py | 5 ++- src/fuzzfetch/core.py | 72 ++++++++++++++++++++++--------- src/fuzzfetch/extract.py | 28 ++++++++---- src/fuzzfetch/models.py | 93 ++++++++++++++++++++++++++++------------ tests/test_core.py | 10 +++-- tests/test_fetch.py | 2 +- tests/test_models.py | 6 ++- 7 files changed, 153 insertions(+), 63 deletions(-) diff --git a/src/fuzzfetch/args.py b/src/fuzzfetch/args.py index e93d3bf5..e120df97 100644 --- a/src/fuzzfetch/args.py +++ b/src/fuzzfetch/args.py @@ -55,7 +55,7 @@ def __init__(self) -> None: nargs="*", default=[], help="Specify the build artifacts to download. " - "Valid options: firefox js common gtest mozharness searchfox " + "Valid options: firefox js common gtest mozharness searchfox thunderbird " f"(default: {' '.join(FetcherArgs.DEFAULT_TARGETS)})", ) target_group.add_argument( @@ -186,6 +186,9 @@ def sanity_check(self, args: Namespace) -> None: if args.branch is None: args.branch = "central" + if "thunderbird" in args.target and len(args.target) > 1: + self.parser.error("Cannot specify multiple targets with thunderbird") + if "firefox" in args.target and args.fuzzilli: self.parser.error("Cannot specify --target firefox and --fuzzilli") diff --git a/src/fuzzfetch/core.py b/src/fuzzfetch/core.py index a9250b08..0ce5c0e3 100644 --- a/src/fuzzfetch/core.py +++ b/src/fuzzfetch/core.py @@ -33,7 +33,14 @@ from .download import download_url, get_url, resolve_url from .errors import FetcherException from .extract import extract_dmg, extract_tar, extract_zip -from .models import BuildFlags, BuildSearchOrder, BuildTask, HgRevision, Platform +from .models import ( + BuildFlags, + BuildSearchOrder, + BuildTask, + HgRevision, + Platform, + Product, +) from .path import PathArg from .path import rmtree as junction_rmtree from .utils import _create_utc_datetime, is_date, is_namespace, is_rev @@ -84,6 +91,7 @@ def __init__( flags: Sequence[bool] | BuildFlags, targets: Sequence[str], platform: Platform | None = None, + product: str | Product = "firefox", simulated: str | None = None, nearest: BuildSearchOrder | None = None, ) -> None: @@ -106,6 +114,7 @@ def __init__( self._platform = platform or Platform() self._simulated = simulated self._targets = targets + self._product = product if isinstance(product, Product) else Product(product) self._task: BuildTask | None = None if not isinstance(build, BuildTask): @@ -126,6 +135,7 @@ def __init__( build, branch, self._flags, + self._product, platform=self._platform, simulated=self._simulated, ), @@ -155,7 +165,7 @@ def __init__( date = datetime.strptime(build, "%Y%m%d%H%M%S") requested = timezone("UTC").localize(date) elif is_rev(build): - requested = HgRevision(build, branch).pushdate + requested = HgRevision(build, branch, self._product).pushdate else: # If no match, assume it's a TaskCluster namespace if re.match(r".*[0-9]{4}\.[0-9]{2}\.[0-9]{2}.*", build) is not None: @@ -166,7 +176,9 @@ def __init__( elif re.match(r".*revision.*[0-9[a-f]{40}", build): match = re.search(r"[0-9[a-f]{40}", build) assert match is not None - requested = HgRevision(match.group(0), branch).pushdate + requested = HgRevision( + match.group(0), branch, self._product + ).pushdate assert isinstance(requested, datetime) # If start date is outside the range of the newest/oldest available @@ -198,6 +210,7 @@ def __init__( search_build, branch, self._flags, + self._product, self._platform, self._simulated, ), @@ -244,19 +257,27 @@ def __init__( if self._branch in {"autoland", "try"}: branch = self._branch else: - branch = f"m-{self._branch[0]}" + namespace_initial = ( + self._product.namespace[0] + if self._product.namespace is not None + else "" + ) + branch = f"{namespace_initial}-{self._branch[0]}" self._auto_name = ( f"{self._platform.auto_name_prefix()}{branch}-{self.id}{options}" ) @staticmethod - def resolve_esr(branch: str) -> str: + def resolve_esr(branch: str, product: str) -> str: """Retrieve esr version based on keyword""" if branch not in {"esr-stable", "esr-next"}: raise FetcherException(f"Invalid ESR branch specified: {branch}") - resp = get_url("https://product-details.mozilla.org/1.0/firefox_versions.json") - key = "FIREFOX_ESR" if branch == "esr-stable" else "FIREFOX_ESR_NEXT" + resp = get_url( + f"https://product-details.mozilla.org/1.0/{product}_versions.json" + ) + prefix = product.upper() + key = f"{prefix}_ESR" if branch == "esr-stable" else f"{prefix}_ESR_NEXT" match = re.search(r"^\d+", resp.json()[key]) if match is None: raise FetcherException(f"Unable to identify ESR version for {branch}") @@ -391,13 +412,13 @@ def resolve_targets(self, targets: Sequence[str]) -> None: targets_remaining.remove("js") resolve_url(self.artifact_url("jsshell.zip")) - if "firefox" in targets_remaining: + if self._product.name in targets_remaining: have_exec = True # We only check that crashreporter symbols exist for builds where it is - # enabled and only if downloading firefox itself. + # enabled and only if downloading firefox/thunderbird itself. # Add --disable-crashreporter to mozconfig if you don't need them. syms_wanted = bool(self.moz_info["crashreporter"]) - targets_remaining.remove("firefox") + targets_remaining.remove(self._product.name) if self._platform.system == "Linux": for ext in ("xz", "bz2"): url = self.artifact_url(f"tar.{ext}") @@ -410,7 +431,7 @@ def resolve_targets(self, targets: Sequence[str]) -> None: resolve_url(self.artifact_url("dmg")) elif self._platform.system == "Windows": resolve_url(self.artifact_url("zip")) - elif self._platform.system == "Android": + elif self._platform.system == "Android" and self._product.name == "firefox": artifact_path = "/".join(self._artifact_base.split("/")[:-1]) url = f"{self._artifacts_url}/{artifact_path}/geckoview_example.apk" resolve_url(url) @@ -469,10 +490,10 @@ def extract_build(self, path: PathArg) -> None: self.extract_zip(self.artifact_url("jsshell.zip"), _path) self._write_fuzzmanagerconf("js", path) - if "firefox" in targets_remaining: - targets_remaining.remove("firefox") + if self._product.name in targets_remaining: + targets_remaining.remove(self._product.name) # We only check that crashreporter symbols exist for builds where it is - # enabled and only if downloading firefox itself. + # enabled and only if downloading firefox/thunderbird itself. # Add --disable-crashreporter to mozconfig if you don't need them. syms_wanted = bool(self.moz_info["crashreporter"]) have_exec = True @@ -487,13 +508,13 @@ def extract_build(self, path: PathArg) -> None: self.extract_dmg(path) elif self._platform.system == "Windows": self.extract_zip(self.artifact_url("zip"), path) - elif self._platform.system == "Android": + elif self._platform.system == "Android" and self._product.name == "firefox": self.download_apk(path) else: raise FetcherException( f"'{self._platform.system}' is not a supported platform" ) - self._write_fuzzmanagerconf("firefox", path) + self._write_fuzzmanagerconf(self._product.name, path) if "gtest" in targets_remaining: targets_remaining.remove("gtest") @@ -580,7 +601,7 @@ def _write_fuzzmanagerconf(self, target: str, path: Path) -> None: processor = self._platform.machine assert isinstance(processor, str) output.set("Main", "platform", processor.replace("_", "-")) - output.set("Main", "product", f"mozilla-{self._branch}") + output.set("Main", "product", f"{self._product.prefix}{self._branch}") output.set("Main", "product_version", f"{self.id:.8}-{self.changeset:.12}") if self._platform.system == "Android": output.set("Main", "os", "android") @@ -631,14 +652,14 @@ def extract_zip(self, url: str, path: PathArg = ".") -> None: try: download_url(url, zip_fn) LOG.info(".. extracting") - extract_zip(zip_fn, path) + extract_zip(zip_fn, path, self._product.name) finally: os.unlink(zip_fn) def extract_tar(self, url: str, path: PathArg = ".") -> None: """ Extract builds with .tar.(*) extension - When unpacking a build archive, only extract the firefox directory + When unpacking a build archive, only extract the product directory Arguments: url: artifact to download @@ -650,7 +671,7 @@ def extract_tar(self, url: str, path: PathArg = ".") -> None: try: download_url(url, tar_fn) LOG.info(".. extracting") - extract_tar(tar_fn, mode, path) + extract_tar(tar_fn, mode, path, self._product.name) finally: os.unlink(tar_fn) @@ -722,13 +743,21 @@ def from_args( ) args = parser.parse_args(argv) + product = ( + Product("thunderbird") + if "thunderbird" in args.target + else Product("firefox") + ) + # do this default manually so we can error if combined with --build namespace # parser.set_defaults(branch='central') if not is_namespace(args.build): if args.branch is None: args.branch = "central" elif args.branch.startswith("esr"): - args.branch = Fetcher.resolve_esr(args.branch) + if product.name is None: + raise AttributeError("no product name found for ESR branch") + args.branch = Fetcher.resolve_esr(args.branch, product.name) flags = BuildFlags( args.asan, @@ -749,6 +778,7 @@ def from_args( flags, args.target, platform=Platform(args.os, args.cpu), + product=product, simulated=args.sim, nearest=args.nearest, ) diff --git a/src/fuzzfetch/extract.py b/src/fuzzfetch/extract.py index 3d3ca442..b67fc9b6 100644 --- a/src/fuzzfetch/extract.py +++ b/src/fuzzfetch/extract.py @@ -25,7 +25,9 @@ EXT_WARNINGS = {"bz2", "xz", "zst"} -def extract_zip(zip_fn: PathArg, path: PathArg = ".") -> None: +def extract_zip( + zip_fn: PathArg, path: PathArg = ".", product_name: str | None = "firefox" +) -> None: """Download and extract a zip artifact Arguments: @@ -33,15 +35,17 @@ def extract_zip(zip_fn: PathArg, path: PathArg = ".") -> None: path: where to extract zip contents """ dest_path = Path(path) + if product_name is None: + product_name = "firefox" - def _extract_entry(zip_fp: ZipFile, info: ZipInfo) -> None: + def _extract_entry(zip_fp: ZipFile, info: ZipInfo, product_name: str) -> None: """Extract entries while explicitly setting the proper permissions""" rel_path = Path(info.filename) # strip leading "firefox" from path if rel_path.parts[0] == ".": rel_path = Path(*rel_path.parts[1:]) - if rel_path.parts[0] == "firefox": + if rel_path.parts[0] == product_name: rel_path = Path(*rel_path.parts[1:]) out_path = dest_path / rel_path @@ -59,7 +63,7 @@ def _extract_entry(zip_fp: ZipFile, info: ZipInfo) -> None: with ZipFile(zip_fn) as zip_fp: for info in zip_fp.infolist(): - _extract_entry(zip_fp, info) + _extract_entry(zip_fp, info, product_name) def _is_within_directory(directory: PathArg, target: PathArg) -> bool: @@ -71,7 +75,12 @@ def _is_within_directory(directory: PathArg, target: PathArg) -> bool: return prefix == abs_directory -def extract_tar(tar_fn: PathArg, mode: str = "", path: PathArg = ".") -> None: +def extract_tar( + tar_fn: PathArg, + mode: str = "", + path: PathArg = ".", + product_name: str | None = "firefox", +) -> None: """Extract builds with .tar.(*) extension When unpacking a build archive, only extract the firefox directory @@ -79,8 +88,11 @@ def extract_tar(tar_fn: PathArg, mode: str = "", path: PathArg = ".") -> None: tar_fn: path to tar archive mode: compression type path: where to extract tar contents + product_name: name of the target product """ tmp_fn = None + if product_name is None: + product_name = "firefox" try: def _external_decomp(decomp: str) -> None: @@ -135,10 +147,10 @@ def _external_decomp(decomp: str) -> None: for member in tar.getmembers(): if not _is_within_directory(path, Path(path) / member.name): raise RuntimeError("Attempted Path Traversal in Tar File") - if member.name.startswith("firefox/"): - member.name = member.name[8:] + if member.name.startswith(product_name + "/"): + member.name = member.name[len(product_name) + 1 :] members.append(member) - elif member.name != "firefox": + elif member.name != product_name: # Ignore top-level build directory members.append(member) tar.extractall(members=members, path=path) diff --git a/src/fuzzfetch/models.py b/src/fuzzfetch/models.py index 7eba4a41..f4f19b2d 100644 --- a/src/fuzzfetch/models.py +++ b/src/fuzzfetch/models.py @@ -9,7 +9,8 @@ from datetime import datetime from enum import Enum from functools import total_ordering -from itertools import chain, product +from itertools import chain +from itertools import product as cproduct from logging import getLogger from platform import machine as plat_machine from platform import system as plat_system @@ -131,6 +132,7 @@ def iterall( build: str, branch: str, flags: BuildFlags, + product: Product, platform: Platform | None = None, simulated: str | None = None, ) -> Iterator[BuildTask]: @@ -151,7 +153,7 @@ def iterall( task_template_paths: Iterable[tuple[str, str]] = tuple( (template, f"{path}{flag_str}") for (template, path) in cls._pushdate_template_paths( - build_date_ns, branch, target_platform + build_date_ns, branch, product, target_platform ) if filt in path ) @@ -159,25 +161,28 @@ def iterall( elif is_rev(build): # If a short hash was supplied, resolve it to a long one. if len(build) == 12: - build = HgRevision(build, branch).hash + build = HgRevision(build, branch, product).hash flag_str = flags.build_string() task_paths = tuple( f"{path}{flag_str}" - for path in cls._revision_paths(build.lower(), branch, target_platform) + for path in cls._revision_paths( + build.lower(), branch, product, target_platform + ) ) - task_template_paths = product((cls.TASKCLUSTER_API,), task_paths) + task_template_paths = cproduct((cls.TASKCLUSTER_API,), task_paths) elif build == "latest": - if branch not in {"autoland", "try"}: - branch = f"mozilla-{branch}" + branch = product.prefix_branch(branch) namespaces = [] if not any(flags): # Opt builds are now indexed under 'shippable' - namespaces.append(f"gecko.v2.{branch}.shippable.latest") - namespaces.append(f"gecko.v2.{branch}.latest") + namespaces.append(f"{product.namespace}.v2.{branch}.shippable.latest") + namespaces.append(f"{product.namespace}.v2.{branch}.latest") - prod = "mobile" if "android" in target_platform else "firefox" + prod = "mobile" if "android" in target_platform else product.name + if prod is None: + raise AttributeError("no product name found for task path") suffix = f"{target_platform}{flags.build_string()}" def generate_task_paths( @@ -194,7 +199,7 @@ def generate_task_paths( yield f"/task/{namespace}.{prod_}.sm-{suffix_}" task_paths = tuple(generate_task_paths(namespaces, prod, suffix, simulated)) - task_template_paths = product((cls.TASKCLUSTER_API,), task_paths) + task_template_paths = cproduct((cls.TASKCLUSTER_API,), task_paths) else: # try to use build argument directly as a namespace @@ -213,7 +218,7 @@ def generate_task_paths( ) ) - for template_path, try_wo_opt in product(task_template_paths, (False, True)): + for template_path, try_wo_opt in cproduct(task_template_paths, (False, True)): template, path = template_path if try_wo_opt: @@ -264,17 +269,17 @@ def _pushdate_template_paths( cls, pushdate: str, branch: str, + product: Product, target_platform: str, ) -> Iterator[tuple[str, str]]: """Multiple entries exist per push date. Iterate over all until a working entry is found """ - if branch not in {"autoland", "try"}: - branch = f"mozilla-{branch}" + branch = product.prefix_branch(branch) paths = ( - f"/namespaces/gecko.v2.{branch}.shippable.{pushdate}", - f"/namespaces/gecko.v2.{branch}.pushdate.{pushdate}", + f"/namespaces/{product.namespace}.v2.{branch}.shippable.{pushdate}", + f"/namespaces/{product.namespace}.v2.{branch}.pushdate.{pushdate}", ) for path in paths: @@ -286,33 +291,33 @@ def _pushdate_template_paths( except RequestException: continue - prod = "mobile" if "android" in target_platform else "firefox" + prod = "mobile" if "android" in target_platform else product.name json = base.json() for namespace in sorted(json["namespaces"], key=lambda x: str(x["name"])): task_paths = ( f"/task/{namespace['namespace']}.{prod}.{target_platform}", f"/task/{namespace['namespace']}.{prod}.sm-{target_platform}", ) - yield from product((cls.TASKCLUSTER_API,), task_paths) + yield from cproduct((cls.TASKCLUSTER_API,), task_paths) @classmethod def _revision_paths( cls, rev: str, branch: str, + product: Product, target_platform: str, ) -> Iterator[str]: """Retrieve the API path for revision based builds""" - if branch not in {"autoland", "try"}: - branch = f"mozilla-{branch}" + branch = product.prefix_branch(branch) namespaces = ( - f"gecko.v2.{branch}.shippable.revision.{rev}", - f"gecko.v2.{branch}.revision.{rev}", + f"{product.namespace}.v2.{branch}.shippable.revision.{rev}", + f"{product.namespace}.v2.{branch}.revision.{rev}", ) for namespace in namespaces: - prod = "mobile" if "android" in target_platform else "firefox" + prod = "mobile" if "android" in target_platform else product.name yield f"/task/{namespace}.{prod}.{target_platform}" yield f"/task/{namespace}.{prod}.sm-{target_platform}" @@ -320,7 +325,7 @@ def _revision_paths( class HgRevision: """Class representing a Mercurial revision.""" - def __init__(self, revision: str, branch: str) -> None: + def __init__(self, revision: str, branch: str, product: Product) -> None: """Create a Mercurial revision object. Arguments: @@ -333,9 +338,9 @@ def __init__(self, revision: str, branch: str) -> None: if branch == "autoland": branch = f"integration/{branch}" elif branch in {"release", "beta"} or branch.startswith("esr"): - branch = f"releases/mozilla-{branch}" - elif branch != "try": - branch = f"mozilla-{branch}" + branch = f"releases/{product.prefix}{branch}" + else: + branch = product.prefix_branch(branch) self._data = get_url( f"https://hg.mozilla.org/{branch}/json-rev/{revision}" ).json() @@ -450,3 +455,37 @@ def auto_name_prefix(self) -> str: "android-aarch64": "android-arm64", }.get(self.gecko_platform, self.gecko_platform) return f"{platform}-" + + +class Product: + """Class representing relevant strings for target product (firefox/thunderbird)""" + + PREFIXES = MappingProxyType( + { + "firefox": "mozilla-", + "thunderbird": "comm-", + } + ) + NAMESPACES = MappingProxyType( + { + "firefox": "gecko", + "thunderbird": "comm", + } + ) + + def __init__(self, product: str | None = None): + self.name = product + self.prefix = ( + self.PREFIXES.get(product) if product and product in self.PREFIXES else "" + ) + self.namespace = self.NAMESPACES.get(product) if product else None + + def prefix_branch(self, branch: str) -> str: + """add the appropriate prefix to the branch name (e.g., beta -> mozilla-beta)""" + if self.name == "firefox" and branch not in {"autoland", "try"}: + return f"{self.prefix}{branch}" + if self.name == "thunderbird": + if branch == "try": + return "try-comm-central" + return f"{self.prefix}{branch}" + return branch diff --git a/tests/test_core.py b/tests/test_core.py index 61507917..e8ca5967 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6,7 +6,7 @@ import pytest # pylint: disable=import-error from fuzzfetch.core import Fetcher -from fuzzfetch.models import BuildFlags, BuildTask, Platform +from fuzzfetch.models import BuildFlags, BuildTask, Platform, Product @pytest.mark.vcr() @@ -21,7 +21,9 @@ def fake_extract_tar(_self, _url, path): flags = BuildFlags(debug=True, fuzzing=True) platform = Platform("Linux", "x86_64") - task = next(BuildTask.iterall("latest", "central", flags, platform)) + task = next( + BuildTask.iterall("latest", "central", flags, Product("firefox"), platform) + ) fetcher = Fetcher("central", task, flags, ["firefox"], platform) fetcher.extract_build(tmp_path) assert set(tmp_path.glob("**/*")) == { @@ -50,7 +52,9 @@ def fake_extract_zip(_self, url, path): flags = BuildFlags(debug=True, fuzzing=True) platform = Platform("Darwin", "x86_64") - task = next(BuildTask.iterall("latest", "central", flags, platform)) + task = next( + BuildTask.iterall("latest", "central", flags, Product("firefox"), platform) + ) fetcher = Fetcher("central", task, flags, ["firefox"], platform) fetcher.extract_build(tmp_path) assert set(tmp_path.glob("**/*")) == { diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 5ce7b3e8..3779aeb1 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -148,7 +148,7 @@ def test_metadata(branch, build_flags, os_, cpu, as_args): )[0] else: if branch.startswith("esr"): - branch = Fetcher.resolve_esr(branch) + branch = Fetcher.resolve_esr(branch, "firefox") fetcher = Fetcher( branch, "latest", diff --git a/tests/test_models.py b/tests/test_models.py index d5de049d..9f06153b 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -10,7 +10,7 @@ import pytest # pylint: disable=import-error from fuzzfetch import FetcherException -from fuzzfetch.models import BuildFlags, HgRevision, Platform +from fuzzfetch.models import BuildFlags, HgRevision, Platform, Product def test_build_flags_custom_initialization(): @@ -97,7 +97,9 @@ def test_build_flags_update_raises_on_mismatch(initial, build_string, missing): ) def test_hgrevision_properties(known_branch, known_revision): """Test HgRevision properties with a real revision and branch.""" - revision = HgRevision(revision=known_revision, branch=known_branch) + revision = HgRevision( + revision=known_revision, branch=known_branch, product=Product("firefox") + ) assert isinstance(revision.pushdate, datetime) assert revision.pushdate.tzinfo is not None From 78fd9bd6d3e9ba6ffe4e88f89a25b014369eb090 Mon Sep 17 00:00:00 2001 From: Jesse Schwartzentruber Date: Thu, 6 Nov 2025 16:53:23 -0500 Subject: [PATCH 2/3] fix: broken type annotation in 3.9 --- src/fuzzfetch/extract.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fuzzfetch/extract.py b/src/fuzzfetch/extract.py index b67fc9b6..9da7f7b1 100644 --- a/src/fuzzfetch/extract.py +++ b/src/fuzzfetch/extract.py @@ -3,6 +3,8 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. """Code for extracting archives""" +from __future__ import annotations + import os from logging import getLogger from os.path import abspath, commonpath From 24883d1d615982245b70e0199a51b677e9730218 Mon Sep 17 00:00:00 2001 From: Jesse Schwartzentruber Date: Thu, 6 Nov 2025 16:54:31 -0500 Subject: [PATCH 3/3] feat: add assertion for hg rev lookup --- src/fuzzfetch/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/fuzzfetch/models.py b/src/fuzzfetch/models.py index f4f19b2d..b667acb3 100644 --- a/src/fuzzfetch/models.py +++ b/src/fuzzfetch/models.py @@ -336,6 +336,10 @@ def __init__(self, revision: str, branch: str, product: Product) -> None: raise FetcherException(f"Can't lookup revision date for branch: {branch}") if branch == "autoland": + if product.name != "firefox": + raise FetcherException( + f"Can't lookup autoland revision for product: {product}" + ) branch = f"integration/{branch}" elif branch in {"release", "beta"} or branch.startswith("esr"): branch = f"releases/{product.prefix}{branch}"