Add Thunderbird

src/fuzzfetch/args.py

@@ -55,7 +55,7 @@ def __init__(self) -> None:
             nargs="*",
             default=[],
             help="Specify the build artifacts to download. "
-            "Valid options: firefox js common gtest mozharness searchfox "
+            "Valid options: firefox js common gtest mozharness searchfox thunderbird "
             f"(default: {' '.join(FetcherArgs.DEFAULT_TARGETS)})",
         )
         target_group.add_argument(
@@ -186,6 +186,9 @@ def sanity_check(self, args: Namespace) -> None:
         if args.branch is None:
             args.branch = "central"
 
+        if "thunderbird" in args.target and len(args.target) > 1:
+            self.parser.error("Cannot specify multiple targets with thunderbird")
+
         if "firefox" in args.target and args.fuzzilli:
             self.parser.error("Cannot specify --target firefox and --fuzzilli")
 

src/fuzzfetch/core.py

@@ -33,7 +33,14 @@
 from .download import download_url, get_url, resolve_url
 from .errors import FetcherException
 from .extract import extract_dmg, extract_tar, extract_zip
-from .models import BuildFlags, BuildSearchOrder, BuildTask, HgRevision, Platform
+from .models import (
+    BuildFlags,
+    BuildSearchOrder,
+    BuildTask,
+    HgRevision,
+    Platform,
+    Product,
+)
 from .path import PathArg
 from .path import rmtree as junction_rmtree
 from .utils import _create_utc_datetime, is_date, is_namespace, is_rev
@@ -84,6 +91,7 @@ def __init__(
         flags: Sequence[bool] | BuildFlags,
         targets: Sequence[str],
         platform: Platform | None = None,
+        product: str | Product = "firefox",
         simulated: str | None = None,
         nearest: BuildSearchOrder | None = None,
     ) -> None:
@@ -106,6 +114,7 @@ def __init__(
         self._platform = platform or Platform()
         self._simulated = simulated
         self._targets = targets
+        self._product = product if isinstance(product, Product) else Product(product)
         self._task: BuildTask | None = None
 
         if not isinstance(build, BuildTask):
@@ -126,6 +135,7 @@ def __init__(
                         build,
                         branch,
                         self._flags,
+                        self._product,
                         platform=self._platform,
                         simulated=self._simulated,
                     ),
@@ -155,7 +165,7 @@ def __init__(
                         date = datetime.strptime(build, "%Y%m%d%H%M%S")
                     requested = timezone("UTC").localize(date)
                 elif is_rev(build):
-                    requested = HgRevision(build, branch).pushdate
+                    requested = HgRevision(build, branch, self._product).pushdate
                 else:
                     # If no match, assume it's a TaskCluster namespace
                     if re.match(r".*[0-9]{4}\.[0-9]{2}\.[0-9]{2}.*", build) is not None:
@@ -166,7 +176,9 @@ def __init__(
                     elif re.match(r".*revision.*[0-9[a-f]{40}", build):
                         match = re.search(r"[0-9[a-f]{40}", build)
                         assert match is not None
-                        requested = HgRevision(match.group(0), branch).pushdate
+                        requested = HgRevision(
+                            match.group(0), branch, self._product
+                        ).pushdate
                 assert isinstance(requested, datetime)
 
                 # If start date is outside the range of the newest/oldest available
@@ -198,6 +210,7 @@ def __init__(
                                 search_build,
                                 branch,
                                 self._flags,
+                                self._product,
                                 self._platform,
                                 self._simulated,
                             ),
@@ -244,19 +257,27 @@ def __init__(
         if self._branch in {"autoland", "try"}:
             branch = self._branch
         else:
-            branch = f"m-{self._branch[0]}"
+            namespace_initial = (
+                self._product.namespace[0]
+                if self._product.namespace is not None
+                else ""
+            )
+            branch = f"{namespace_initial}-{self._branch[0]}"
         self._auto_name = (
             f"{self._platform.auto_name_prefix()}{branch}-{self.id}{options}"
         )
 
     @staticmethod
-    def resolve_esr(branch: str) -> str:
+    def resolve_esr(branch: str, product: str) -> str:
         """Retrieve esr version based on keyword"""
         if branch not in {"esr-stable", "esr-next"}:
             raise FetcherException(f"Invalid ESR branch specified: {branch}")
 
-        resp = get_url("https://product-details.mozilla.org/1.0/firefox_versions.json")
-        key = "FIREFOX_ESR" if branch == "esr-stable" else "FIREFOX_ESR_NEXT"
+        resp = get_url(
+            f"https://product-details.mozilla.org/1.0/{product}_versions.json"
+        )
+        prefix = product.upper()
+        key = f"{prefix}_ESR" if branch == "esr-stable" else f"{prefix}_ESR_NEXT"
         match = re.search(r"^\d+", resp.json()[key])
         if match is None:
             raise FetcherException(f"Unable to identify ESR version for {branch}")
@@ -391,13 +412,13 @@ def resolve_targets(self, targets: Sequence[str]) -> None:
             targets_remaining.remove("js")
             resolve_url(self.artifact_url("jsshell.zip"))
 
-        if "firefox" in targets_remaining:
+        if self._product.name in targets_remaining:
             have_exec = True
             # We only check that crashreporter symbols exist for builds where it is
-            # enabled and only if downloading firefox itself.
+            # enabled and only if downloading firefox/thunderbird itself.
             # Add --disable-crashreporter to mozconfig if you don't need them.
             syms_wanted = bool(self.moz_info["crashreporter"])
-            targets_remaining.remove("firefox")
+            targets_remaining.remove(self._product.name)
             if self._platform.system == "Linux":
                 for ext in ("xz", "bz2"):
                     url = self.artifact_url(f"tar.{ext}")
@@ -410,7 +431,7 @@ def resolve_targets(self, targets: Sequence[str]) -> None:
                 resolve_url(self.artifact_url("dmg"))
             elif self._platform.system == "Windows":
                 resolve_url(self.artifact_url("zip"))
-            elif self._platform.system == "Android":
+            elif self._platform.system == "Android" and self._product.name == "firefox":
                 artifact_path = "/".join(self._artifact_base.split("/")[:-1])
                 url = f"{self._artifacts_url}/{artifact_path}/geckoview_example.apk"
                 resolve_url(url)
@@ -469,10 +490,10 @@ def extract_build(self, path: PathArg) -> None:
             self.extract_zip(self.artifact_url("jsshell.zip"), _path)
             self._write_fuzzmanagerconf("js", path)
 
-        if "firefox" in targets_remaining:
-            targets_remaining.remove("firefox")
+        if self._product.name in targets_remaining:
+            targets_remaining.remove(self._product.name)
             # We only check that crashreporter symbols exist for builds where it is
-            # enabled and only if downloading firefox itself.
+            # enabled and only if downloading firefox/thunderbird itself.
             # Add --disable-crashreporter to mozconfig if you don't need them.
             syms_wanted = bool(self.moz_info["crashreporter"])
             have_exec = True
@@ -487,13 +508,13 @@ def extract_build(self, path: PathArg) -> None:
                 self.extract_dmg(path)
             elif self._platform.system == "Windows":
                 self.extract_zip(self.artifact_url("zip"), path)
-            elif self._platform.system == "Android":
+            elif self._platform.system == "Android" and self._product.name == "firefox":
                 self.download_apk(path)
             else:
                 raise FetcherException(
                     f"'{self._platform.system}' is not a supported platform"
                 )
-            self._write_fuzzmanagerconf("firefox", path)
+            self._write_fuzzmanagerconf(self._product.name, path)
 
         if "gtest" in targets_remaining:
             targets_remaining.remove("gtest")
@@ -580,7 +601,7 @@ def _write_fuzzmanagerconf(self, target: str, path: Path) -> None:
         processor = self._platform.machine
         assert isinstance(processor, str)
         output.set("Main", "platform", processor.replace("_", "-"))
-        output.set("Main", "product", f"mozilla-{self._branch}")
+        output.set("Main", "product", f"{self._product.prefix}{self._branch}")
         output.set("Main", "product_version", f"{self.id:.8}-{self.changeset:.12}")
         if self._platform.system == "Android":
             output.set("Main", "os", "android")
@@ -631,14 +652,14 @@ def extract_zip(self, url: str, path: PathArg = ".") -> None:
         try:
             download_url(url, zip_fn)
             LOG.info(".. extracting")
-            extract_zip(zip_fn, path)
+            extract_zip(zip_fn, path, self._product.name)
         finally:
             os.unlink(zip_fn)
 
     def extract_tar(self, url: str, path: PathArg = ".") -> None:
         """
         Extract builds with .tar.(*) extension
-        When unpacking a build archive, only extract the firefox directory
+        When unpacking a build archive, only extract the product directory
 
         Arguments:
             url: artifact to download
@@ -650,7 +671,7 @@ def extract_tar(self, url: str, path: PathArg = ".") -> None:
         try:
             download_url(url, tar_fn)
             LOG.info(".. extracting")
-            extract_tar(tar_fn, mode, path)
+            extract_tar(tar_fn, mode, path, self._product.name)
         finally:
             os.unlink(tar_fn)
 
@@ -722,13 +743,21 @@ def from_args(
         )
         args = parser.parse_args(argv)
 
+        product = (
+            Product("thunderbird")
+            if "thunderbird" in args.target
+            else Product("firefox")
+        )
+
         # do this default manually so we can error if combined with --build namespace
         # parser.set_defaults(branch='central')
         if not is_namespace(args.build):
             if args.branch is None:
                 args.branch = "central"
             elif args.branch.startswith("esr"):
-                args.branch = Fetcher.resolve_esr(args.branch)
+                if product.name is None:
+                    raise AttributeError("no product name found for ESR branch")
+                args.branch = Fetcher.resolve_esr(args.branch, product.name)
 
         flags = BuildFlags(
             args.asan,
@@ -749,6 +778,7 @@ def from_args(
             flags,
             args.target,
             platform=Platform(args.os, args.cpu),
+            product=product,
             simulated=args.sim,
             nearest=args.nearest,
         )

src/fuzzfetch/extract.py

@@ -3,6 +3,8 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 """Code for extracting archives"""
 
+from __future__ import annotations
+
 import os
 from logging import getLogger
 from os.path import abspath, commonpath
@@ -25,23 +27,27 @@
 EXT_WARNINGS = {"bz2", "xz", "zst"}
 
 
-def extract_zip(zip_fn: PathArg, path: PathArg = ".") -> None:
+def extract_zip(
+    zip_fn: PathArg, path: PathArg = ".", product_name: str | None = "firefox"
+) -> None:
     """Download and extract a zip artifact
 
     Arguments:
         zip_fn: path to zip archive
         path: where to extract zip contents
     """
     dest_path = Path(path)
+    if product_name is None:
+        product_name = "firefox"
 
-    def _extract_entry(zip_fp: ZipFile, info: ZipInfo) -> None:
+    def _extract_entry(zip_fp: ZipFile, info: ZipInfo, product_name: str) -> None:
         """Extract entries while explicitly setting the proper permissions"""
         rel_path = Path(info.filename)
 
         # strip leading "firefox" from path
         if rel_path.parts[0] == ".":
             rel_path = Path(*rel_path.parts[1:])
-        if rel_path.parts[0] == "firefox":
+        if rel_path.parts[0] == product_name:
             rel_path = Path(*rel_path.parts[1:])
 
         out_path = dest_path / rel_path
@@ -59,7 +65,7 @@ def _extract_entry(zip_fp: ZipFile, info: ZipInfo) -> None:
 
     with ZipFile(zip_fn) as zip_fp:
         for info in zip_fp.infolist():
-            _extract_entry(zip_fp, info)
+            _extract_entry(zip_fp, info, product_name)
 
 
 def _is_within_directory(directory: PathArg, target: PathArg) -> bool:
@@ -71,16 +77,24 @@ def _is_within_directory(directory: PathArg, target: PathArg) -> bool:
     return prefix == abs_directory
 
 
-def extract_tar(tar_fn: PathArg, mode: str = "", path: PathArg = ".") -> None:
+def extract_tar(
+    tar_fn: PathArg,
+    mode: str = "",
+    path: PathArg = ".",
+    product_name: str | None = "firefox",
+) -> None:
     """Extract builds with .tar.(*) extension
     When unpacking a build archive, only extract the firefox directory
 
     Arguments:
         tar_fn: path to tar archive
         mode: compression type
         path: where to extract tar contents
+        product_name: name of the target product
     """
     tmp_fn = None
+    if product_name is None:
+        product_name = "firefox"
     try:
 
         def _external_decomp(decomp: str) -> None:
@@ -135,10 +149,10 @@ def _external_decomp(decomp: str) -> None:
             for member in tar.getmembers():
                 if not _is_within_directory(path, Path(path) / member.name):
                     raise RuntimeError("Attempted Path Traversal in Tar File")
-                if member.name.startswith("firefox/"):
-                    member.name = member.name[8:]
+                if member.name.startswith(product_name + "/"):
+                    member.name = member.name[len(product_name) + 1 :]
                     members.append(member)
-                elif member.name != "firefox":
+                elif member.name != product_name:
                     # Ignore top-level build directory
                     members.append(member)
             tar.extractall(members=members, path=path)