From 71f167ec757cfe9277b322fadb85cdb39a6450fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 17:32:57 +0100 Subject: [PATCH 01/48] Add authorship information to Druid 26.0.0 patches --- druid/stackable/patches/26.0.0/07-cyclonedx-plugin.patch | 4 ++++ .../26.0.0/08-CVE-2024-36114-bump-aircompressor-0-27.patch | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/druid/stackable/patches/26.0.0/07-cyclonedx-plugin.patch b/druid/stackable/patches/26.0.0/07-cyclonedx-plugin.patch index 1635101fa..c726c4a5e 100644 --- a/druid/stackable/patches/26.0.0/07-cyclonedx-plugin.patch +++ b/druid/stackable/patches/26.0.0/07-cyclonedx-plugin.patch @@ -1,3 +1,7 @@ +Update CycloneDX plugin + +From: Lukas Voetmand +--- diff --git a/pom.xml b/pom.xml index c0f0654..133cbf8 100644 --- a/pom.xml diff --git a/druid/stackable/patches/26.0.0/08-CVE-2024-36114-bump-aircompressor-0-27.patch b/druid/stackable/patches/26.0.0/08-CVE-2024-36114-bump-aircompressor-0-27.patch index b5fb91b5f..2062b4e46 100644 --- a/druid/stackable/patches/26.0.0/08-CVE-2024-36114-bump-aircompressor-0-27.patch +++ b/druid/stackable/patches/26.0.0/08-CVE-2024-36114-bump-aircompressor-0-27.patch @@ -1,5 +1,8 @@ Fix CVE-2024-36114 -see https://github.com/stackabletech/vulnerabilities/issues/834 + +From: Malte Sander + +See https://github.com/stackabletech/vulnerabilities/issues/834 Aircompressor is a library with ports of the Snappy, LZO, LZ4, and Zstandard compression algorithms to Java. All decompressor @@ -17,7 +20,7 @@ have been fixed. When decompressing data from untrusted users, this can be exploited for a denial-of-service attack by crashing the JVM, or to leak other sensitive information from the Java process. There are no known workarounds for this issue. - +--- diff --git a/pom.xml b/pom.xml index c0f06547f8..f1c6e2f9ee 100644 --- a/pom.xml From a6be1285a002ae3187331c86850d62ce03198569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 17:41:02 +0100 Subject: [PATCH 02/48] Add patchable checkout script --- .gitignore | 1 + druid/stackable/patches/26.0.0/patchable.toml | 2 + patchable.nu | 90 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 druid/stackable/patches/26.0.0/patchable.toml create mode 100755 patchable.nu diff --git a/.gitignore b/.gitignore index c678a5e10..44e2c49f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] +patchable-work/ diff --git a/druid/stackable/patches/26.0.0/patchable.toml b/druid/stackable/patches/26.0.0/patchable.toml new file mode 100644 index 000000000..264c71a6a --- /dev/null +++ b/druid/stackable/patches/26.0.0/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/apache/druid.git" +base = "7cffb81a8e124d5f218f9af16ad685acf5e9c67c" diff --git a/patchable.nu b/patchable.nu new file mode 100755 index 000000000..1f040c0d9 --- /dev/null +++ b/patchable.nu @@ -0,0 +1,90 @@ +#!/usr/bin/env nu + +use std log + +def repo-root [] { + git rev-parse --show-toplevel +} + +def product-root [product: string] { + $"(repo-root)/($product)" +} + +def product-work-root [product: string] { + $"(product-root $product)/patchable-work" +} + +def product-repo [product: string, --upstream: string] { + let repo_path = $"(product-work-root $product)/product-repo" + log info $"Repository root for ($product) is ($repo_path)" + if not ($repo_path | path exists) { + log info $"Repository root not found, cloning from upstream ($upstream)" + git clone --bare $upstream $repo_path + } else { + log info "Repository root found, reusing" + } + $repo_path +} + +def product-version-worktree-root [product: string, version: string] { + $"(product-work-root $product)/worktree/($version)" +} + +def product-version-worktree-branch [version: string] { + $"patchable/($version)" +} + +def product-version-patch-dir [product: string, version: string] { + $"(product-root $product)/stackable/patches/($version)" +} + +def product-version-config [product: string, version: string] { + let path = $"(product-version-patch-dir $product $version)/patchable.toml" + log info $"Loading patch config from ($path)" + open $path +} + +def "main" [] { + print "Usage:" + print " patchable checkout [--help]" +} + +def "main checkout" [ + product: string + version: string + --force +] { + let config = product-version-config $product $version + let product_repo = (product-repo $product --upstream=$config.upstream) + let worktree_path = product-version-worktree-root $product $version + let worktree_branch = product-version-worktree-branch $version + let $patch_dir = product-version-patch-dir $product $version + log info $"Worktree root is ($worktree_path)" + log info $"Worktree branch is ($worktree_branch), from base ($config.base) and patches at ($patch_dir)" + # These environment variables make git operate on the product worktree from now on + # $GIT_DIR must be the worktree's .git dir, even if that is just an alias for the backing repo, since each worktree maintains its own index + $env.GIT_DIR = $"($worktree_path)/.git" + $env.GIT_WORK_TREE = $worktree_path + let worktree_git_dir = git rev-parse --git-dir + let worktree_rebase_progress_dir = $"($worktree_git_dir)/rebase-apply" + if ($worktree_path | path exists) { + log info "Worktree root already exists, resetting" + if ($worktree_rebase_progress_dir | path exists) { + if $force { + log warning "Rebase/apply is in progress, aborting it" + git am --abort + } else { + error make {msg: "Rebase/apply is in progress, abort manually or pass --force flag"} + } + } + git checkout $config.base + } else { + log info "Worktree root not found, creating" + # $GIT_DIR won't exist yet, so we need to override it + git --git-dir $product_repo --work-tree $product_repo worktree add $worktree_path $config.base --detach + } + log info $"Creating work branch ($worktree_branch)" + git checkout -B $worktree_branch + log info $"Importing patches" + git am $"($patch_dir)/series" --patch-format stgit-series +} From 9e91e7e03f30ef447cdbb62f209121b60091cc02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:12:44 +0100 Subject: [PATCH 03/48] Add patchable export script, convert Druid 26.0.0 patches to patchable --- ...races-of-the-druid-ranger-extension.patch} | 15 ++++++--- ...-Prometheus-emitter-in-distribution.patch} | 12 ++++--- ...003-Stop-building-unused-extensions.patch} | 11 +++++-- ...dencies-that-have-a-new-patch-relea.patch} | 18 ++++++++--- ...e-jackson-dataformat-xml-dependency.patch} | 11 +++++-- ...op-building-the-tar.gz-distribution.patch} | 10 ++++-- ...tch => 0007-Update-CycloneDX-plugin.patch} | 16 +++++++--- ...27.patch => 0008-Fix-CVE-2024-36114.patch} | 15 ++++++--- druid/stackable/patches/26.0.0/series | 9 ------ patchable.nu | 32 +++++++++++++++++-- 10 files changed, 109 insertions(+), 40 deletions(-) rename druid/stackable/patches/26.0.0/{01-remove-ranger-security.patch => 0001-Removes-all-traces-of-the-druid-ranger-extension.patch} (86%) rename druid/stackable/patches/26.0.0/{02-prometheus-emitter-from-source.patch => 0002-Include-Prometheus-emitter-in-distribution.patch} (91%) rename druid/stackable/patches/26.0.0/{03-stop-building-unused-extensions.patch => 0003-Stop-building-unused-extensions.patch} (91%) rename druid/stackable/patches/26.0.0/{04-update-patch-dependencies.patch => 0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch} (89%) rename druid/stackable/patches/26.0.0/{05-xmllayout-dependencies.patch => 0005-Include-jackson-dataformat-xml-dependency.patch} (79%) rename druid/stackable/patches/26.0.0/{06-dont-build-targz.patch => 0006-Stop-building-the-tar.gz-distribution.patch} (76%) rename druid/stackable/patches/26.0.0/{07-cyclonedx-plugin.patch => 0007-Update-CycloneDX-plugin.patch} (68%) rename druid/stackable/patches/26.0.0/{08-CVE-2024-36114-bump-aircompressor-0-27.patch => 0008-Fix-CVE-2024-36114.patch} (86%) delete mode 100644 druid/stackable/patches/26.0.0/series diff --git a/druid/stackable/patches/26.0.0/01-remove-ranger-security.patch b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch similarity index 86% rename from druid/stackable/patches/26.0.0/01-remove-ranger-security.patch rename to druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch index ad5d1a144..7a2f66506 100644 --- a/druid/stackable/patches/26.0.0/01-remove-ranger-security.patch +++ b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch @@ -1,10 +1,12 @@ -Removes all traces of the druid ranger extension - +From 92424a94661977b29d5294a2cd3a38783941e280 Mon Sep 17 00:00:00 2001 From: Lars Francke - +Date: Thu, 12 Dec 2024 17:59:17 +0100 +Subject: [PATCH 1/8] Removes all traces of the druid ranger extension --- - 0 files changed + distribution/pom.xml | 4 ---- + pom.xml | 1 - + 2 files changed, 5 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index eec26171af..a6e72cf2c2 100644 @@ -40,3 +42,8 @@ index 0c6294f5ed..a33c6bd521 100644 extensions-core/druid-catalog extensions-core/testing-tools + +base-commit: 7cffb81a8e124d5f218f9af16ad685acf5e9c67c +-- +2.47.1 + diff --git a/druid/stackable/patches/26.0.0/02-prometheus-emitter-from-source.patch b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch similarity index 91% rename from druid/stackable/patches/26.0.0/02-prometheus-emitter-from-source.patch rename to druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch index 84c042d77..416569b48 100644 --- a/druid/stackable/patches/26.0.0/02-prometheus-emitter-from-source.patch +++ b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch @@ -1,10 +1,11 @@ -Include Prometheus emitter in distribution - +From e215d072b2f4f451a9f8e57db543030e9aae2f52 Mon Sep 17 00:00:00 2001 From: Lars Francke - +Date: Thu, 12 Dec 2024 17:59:17 +0100 +Subject: [PATCH 2/8] Include Prometheus emitter in distribution --- - 0 files changed + distribution/pom.xml | 46 ++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 46 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index a6e72cf2c2..3ab13d5d11 100644 @@ -63,3 +64,6 @@ index a6e72cf2c2..3ab13d5d11 100644 integration-test +-- +2.47.1 + diff --git a/druid/stackable/patches/26.0.0/03-stop-building-unused-extensions.patch b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch similarity index 91% rename from druid/stackable/patches/26.0.0/03-stop-building-unused-extensions.patch rename to druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch index 6674f07ae..52146990e 100644 --- a/druid/stackable/patches/26.0.0/03-stop-building-unused-extensions.patch +++ b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch @@ -1,12 +1,14 @@ -Stop building unused extensions. - +From db629759a1f860d98abb17a6d1dc75c1c9d21e1b Mon Sep 17 00:00:00 2001 From: Lars Francke +Date: Thu, 12 Dec 2024 17:59:17 +0100 +Subject: [PATCH 3/8] Stop building unused extensions. By default Druid builds all community extensions and then discards them while assembling the final distribution. This patch removes unused extensions from the build. --- - 0 files changed + pom.xml | 32 ++++---------------------------- + 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index a33c6bd521..f5001910e1 100644 @@ -67,3 +69,6 @@ index a33c6bd521..f5001910e1 100644 ${repoOrgId} +-- +2.47.1 + diff --git a/druid/stackable/patches/26.0.0/04-update-patch-dependencies.patch b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch similarity index 89% rename from druid/stackable/patches/26.0.0/04-update-patch-dependencies.patch rename to druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch index 3cc1831c9..5a391d9b5 100644 --- a/druid/stackable/patches/26.0.0/04-update-patch-dependencies.patch +++ b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch @@ -1,10 +1,17 @@ -Updates all dependencies that have a new patch release available. - +From d81ea0c7b0daf74d9eb0658b9ffabc61fd2eae78 Mon Sep 17 00:00:00 2001 From: Lars Francke - +Date: Thu, 12 Dec 2024 17:59:17 +0100 +Subject: [PATCH 4/8] Updates all dependencies that have a new patch release + available. --- - 0 files changed + extensions-core/avro-extensions/pom.xml | 2 +- + extensions-core/kubernetes-extensions/pom.xml | 2 +- + extensions-core/orc-extensions/pom.xml | 2 +- + extensions-core/parquet-extensions/pom.xml | 2 +- + extensions-core/protobuf-extensions/pom.xml | 2 +- + pom.xml | 20 +++++++++---------- + 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 35b154a469..a9eb0c6851 100644 @@ -126,3 +133,6 @@ index f5001910e1..2364f27dc4 100644 3.5.10 2.5.7 +-- +2.47.1 + diff --git a/druid/stackable/patches/26.0.0/05-xmllayout-dependencies.patch b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch similarity index 79% rename from druid/stackable/patches/26.0.0/05-xmllayout-dependencies.patch rename to druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch index 112e3b683..cd3aedfce 100644 --- a/druid/stackable/patches/26.0.0/05-xmllayout-dependencies.patch +++ b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch @@ -1,13 +1,15 @@ -Include jackson-dataformat-xml dependency. - +From d8231f06349fc94d7fe633d45b3e6fe1cc7b33e5 Mon Sep 17 00:00:00 2001 From: Lars Francke +Date: Thu, 12 Dec 2024 17:59:17 +0100 +Subject: [PATCH 5/8] Include jackson-dataformat-xml dependency. This allows us to use XmlLayout for Log4jV2. By including it here as a dependency we can make sure that we always have the matching version and we don't need to include it manually later in the build. --- - 0 files changed + server/pom.xml | 5 +++++ + 1 file changed, 5 insertions(+) diff --git a/server/pom.xml b/server/pom.xml index fdc6f1f548..9f18e614e9 100644 @@ -25,3 +27,6 @@ index fdc6f1f548..9f18e614e9 100644 com.fasterxml.jackson.datatype jackson-datatype-joda +-- +2.47.1 + diff --git a/druid/stackable/patches/26.0.0/06-dont-build-targz.patch b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch similarity index 76% rename from druid/stackable/patches/26.0.0/06-dont-build-targz.patch rename to druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch index 1bed79fd1..39605107c 100644 --- a/druid/stackable/patches/26.0.0/06-dont-build-targz.patch +++ b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch @@ -1,11 +1,12 @@ -Stop building the tar.gz distribution. - +From 0e58cf8a9f8bd00e80e4ce3fa53f44e403d223a5 Mon Sep 17 00:00:00 2001 From: Lars Francke +Date: Thu, 12 Dec 2024 17:59:17 +0100 +Subject: [PATCH 6/8] Stop building the tar.gz distribution. All we do is build Druid tar and gzip it only to immediately uncompress it again. So, instead we just skip the compression step entirely. --- - distribution/src/assembly/assembly.xml | 2 +- + distribution/src/assembly/assembly.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/src/assembly/assembly.xml b/distribution/src/assembly/assembly.xml @@ -21,3 +22,6 @@ index ff8e0d2fdd..f9daa49e21 100644 +-- +2.47.1 + diff --git a/druid/stackable/patches/26.0.0/07-cyclonedx-plugin.patch b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch similarity index 68% rename from druid/stackable/patches/26.0.0/07-cyclonedx-plugin.patch rename to druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch index c726c4a5e..6b6cc876d 100644 --- a/druid/stackable/patches/26.0.0/07-cyclonedx-plugin.patch +++ b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch @@ -1,12 +1,17 @@ -Update CycloneDX plugin - +From 71382eac705338ac700b0e347233daacd4e01913 Mon Sep 17 00:00:00 2001 From: Lukas Voetmand +Date: Thu, 12 Dec 2024 17:59:17 +0100 +Subject: [PATCH 7/8] Update CycloneDX plugin + --- + pom.xml | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + diff --git a/pom.xml b/pom.xml -index c0f0654..133cbf8 100644 +index 2364f27dc4..208274a65a 100644 --- a/pom.xml +++ b/pom.xml -@@ -1558,7 +1558,11 @@ +@@ -1533,7 +1533,11 @@ org.cyclonedx cyclonedx-maven-plugin @@ -19,3 +24,6 @@ index c0f0654..133cbf8 100644 package +-- +2.47.1 + diff --git a/druid/stackable/patches/26.0.0/08-CVE-2024-36114-bump-aircompressor-0-27.patch b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch similarity index 86% rename from druid/stackable/patches/26.0.0/08-CVE-2024-36114-bump-aircompressor-0-27.patch rename to druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch index 2062b4e46..74bbba690 100644 --- a/druid/stackable/patches/26.0.0/08-CVE-2024-36114-bump-aircompressor-0-27.patch +++ b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch @@ -1,6 +1,7 @@ -Fix CVE-2024-36114 - +From 0ba7a96e9e089ef35e0f8c6ed7eb99aa56ec67cf Mon Sep 17 00:00:00 2001 From: Malte Sander +Date: Thu, 12 Dec 2024 17:59:17 +0100 +Subject: [PATCH 8/8] Fix CVE-2024-36114 See https://github.com/stackabletech/vulnerabilities/issues/834 @@ -21,11 +22,14 @@ be exploited for a denial-of-service attack by crashing the JVM, or to leak other sensitive information from the Java process. There are no known workarounds for this issue. --- + pom.xml | 6 ++++++ + 1 file changed, 6 insertions(+) + diff --git a/pom.xml b/pom.xml -index c0f06547f8..f1c6e2f9ee 100644 +index 208274a65a..1d53874c41 100644 --- a/pom.xml +++ b/pom.xml -@@ -258,6 +258,12 @@ +@@ -233,6 +233,12 @@ @@ -38,3 +42,6 @@ index c0f06547f8..f1c6e2f9ee 100644 commons-codec +-- +2.47.1 + diff --git a/druid/stackable/patches/26.0.0/series b/druid/stackable/patches/26.0.0/series deleted file mode 100644 index cc7008e05..000000000 --- a/druid/stackable/patches/26.0.0/series +++ /dev/null @@ -1,9 +0,0 @@ -# This series applies on Git commit 7cffb81a8e124d5f218f9af16ad685acf5e9c67c -01-remove-ranger-security.patch -02-prometheus-emitter-from-source.patch -03-stop-building-unused-extensions.patch -04-update-patch-dependencies.patch -05-xmllayout-dependencies.patch -06-dont-build-targz.patch -07-cyclonedx-plugin.patch -08-CVE-2024-36114-bump-aircompressor-0-27.patch diff --git a/patchable.nu b/patchable.nu index 1f040c0d9..a3c7af012 100755 --- a/patchable.nu +++ b/patchable.nu @@ -85,6 +85,34 @@ def "main checkout" [ } log info $"Creating work branch ($worktree_branch)" git checkout -B $worktree_branch - log info $"Importing patches" - git am $"($patch_dir)/series" --patch-format stgit-series + log info "Importing patches" + let series_file = $"($patch_dir)/series" + if ($series_file | path exists) { + log info $"Series file exists at ($series_file), treating as stgit series" + git am $"($patch_dir)/series" --patch-format stgit-series + } else { + log info $"No series file found at ($series_file), treating as git mailbox" + git am ...(glob $"($patch_dir)/*.patch" | sort) + } +} + +def "main export" [ + product: string + version: string +] { + let config = product-version-config $product $version + let product_repo = (product-repo $product --upstream=$config.upstream) + let worktree_path = product-version-worktree-root $product $version + let worktree_branch = product-version-worktree-branch $version + let $patch_dir = product-version-patch-dir $product $version + log info $"Worktree root is ($worktree_path)" + log info $"Worktree branch is ($worktree_branch), from base ($config.base) and patches at ($patch_dir)" + # These environment variables make git operate on the product worktree from now on + # $GIT_DIR must be the worktree's .git dir, even if that is just an alias for the backing repo, since each worktree maintains its own index + $env.GIT_DIR = $"($worktree_path)/.git" + $env.GIT_WORK_TREE = $worktree_path + log info "Deleting existing patches" + rm ...(glob $"($patch_dir)/{*.patch,series}" | tee { print $in }) + log info $"Exporting patches to ($patch_dir)" + git format-patch $config.base -o $patch_dir --base $config.base } From 3796a62de6dde5649214236c6c504ff37f24d2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:25:15 +0100 Subject: [PATCH 04/48] Scrub commit IDs --- ...0001-Removes-all-traces-of-the-druid-ranger-extension.patch | 2 +- .../0002-Include-Prometheus-emitter-in-distribution.patch | 2 +- .../patches/26.0.0/0003-Stop-building-unused-extensions.patch | 2 +- ...-Updates-all-dependencies-that-have-a-new-patch-relea.patch | 2 +- .../0005-Include-jackson-dataformat-xml-dependency.patch | 2 +- .../26.0.0/0006-Stop-building-the-tar.gz-distribution.patch | 2 +- .../patches/26.0.0/0007-Update-CycloneDX-plugin.patch | 2 +- druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch | 2 +- patchable.nu | 3 +++ 9 files changed, 11 insertions(+), 8 deletions(-) diff --git a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch index 7a2f66506..f1deefdf1 100644 --- a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch +++ b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch @@ -1,4 +1,4 @@ -From 92424a94661977b29d5294a2cd3a38783941e280 Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: [PATCH 1/8] Removes all traces of the druid ranger extension diff --git a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch index 416569b48..50f7bea3d 100644 --- a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch +++ b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch @@ -1,4 +1,4 @@ -From e215d072b2f4f451a9f8e57db543030e9aae2f52 Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: [PATCH 2/8] Include Prometheus emitter in distribution diff --git a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch index 52146990e..fda123df0 100644 --- a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch +++ b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch @@ -1,4 +1,4 @@ -From db629759a1f860d98abb17a6d1dc75c1c9d21e1b Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: [PATCH 3/8] Stop building unused extensions. diff --git a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch index 5a391d9b5..b4b4a5496 100644 --- a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch +++ b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch @@ -1,4 +1,4 @@ -From d81ea0c7b0daf74d9eb0658b9ffabc61fd2eae78 Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: [PATCH 4/8] Updates all dependencies that have a new patch release diff --git a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch index cd3aedfce..4b973b42a 100644 --- a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch +++ b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch @@ -1,4 +1,4 @@ -From d8231f06349fc94d7fe633d45b3e6fe1cc7b33e5 Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: [PATCH 5/8] Include jackson-dataformat-xml dependency. diff --git a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch index 39605107c..9defabf62 100644 --- a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch +++ b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch @@ -1,4 +1,4 @@ -From 0e58cf8a9f8bd00e80e4ce3fa53f44e403d223a5 Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: [PATCH 6/8] Stop building the tar.gz distribution. diff --git a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch index 6b6cc876d..e04c92eb8 100644 --- a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch +++ b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch @@ -1,4 +1,4 @@ -From 71382eac705338ac700b0e347233daacd4e01913 Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lukas Voetmand Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: [PATCH 7/8] Update CycloneDX plugin diff --git a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch index 74bbba690..794bcccb9 100644 --- a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch +++ b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch @@ -1,4 +1,4 @@ -From 0ba7a96e9e089ef35e0f8c6ed7eb99aa56ec67cf Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: [PATCH 8/8] Fix CVE-2024-36114 diff --git a/patchable.nu b/patchable.nu index a3c7af012..de48a4a88 100755 --- a/patchable.nu +++ b/patchable.nu @@ -115,4 +115,7 @@ def "main export" [ rm ...(glob $"($patch_dir)/{*.patch,series}" | tee { print $in }) log info $"Exporting patches to ($patch_dir)" git format-patch $config.base -o $patch_dir --base $config.base + # Normally the patches include their own commit IDs, which will change for every for every reimport + log info "Scrubbing commit ID from patches" + sed -i "1s/From [0-9a-f]\\+ /From 0000000000000000000000000000000000000000 /" ...(glob $"($patch_dir)/*.patch") } From 570d0ffcdca273a66f00c22a397270729099fb7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:28:34 +0100 Subject: [PATCH 05/48] Remove patch count from header --- ...0001-Removes-all-traces-of-the-druid-ranger-extension.patch | 2 +- .../0002-Include-Prometheus-emitter-in-distribution.patch | 2 +- .../patches/26.0.0/0003-Stop-building-unused-extensions.patch | 2 +- ...-Updates-all-dependencies-that-have-a-new-patch-relea.patch | 3 +-- .../0005-Include-jackson-dataformat-xml-dependency.patch | 2 +- .../26.0.0/0006-Stop-building-the-tar.gz-distribution.patch | 2 +- .../patches/26.0.0/0007-Update-CycloneDX-plugin.patch | 2 +- druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch | 2 +- patchable.nu | 2 +- 9 files changed, 9 insertions(+), 10 deletions(-) diff --git a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch index f1deefdf1..7b2f55b97 100644 --- a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch +++ b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch @@ -1,7 +1,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 -Subject: [PATCH 1/8] Removes all traces of the druid ranger extension +Subject: Removes all traces of the druid ranger extension --- distribution/pom.xml | 4 ---- diff --git a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch index 50f7bea3d..919f93f94 100644 --- a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch +++ b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch @@ -1,7 +1,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 -Subject: [PATCH 2/8] Include Prometheus emitter in distribution +Subject: Include Prometheus emitter in distribution --- distribution/pom.xml | 46 ++++++++++++++++++++++++++++++++++++++++++++ diff --git a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch index fda123df0..28bd898cd 100644 --- a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch +++ b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch @@ -1,7 +1,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 -Subject: [PATCH 3/8] Stop building unused extensions. +Subject: Stop building unused extensions. By default Druid builds all community extensions and then discards them while assembling the final distribution. This patch removes unused diff --git a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch index b4b4a5496..585225df0 100644 --- a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch +++ b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch @@ -1,8 +1,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 -Subject: [PATCH 4/8] Updates all dependencies that have a new patch release - available. +Subject: Updates all dependencies that have a new patch release available. --- extensions-core/avro-extensions/pom.xml | 2 +- diff --git a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch index 4b973b42a..b62296f5d 100644 --- a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch +++ b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch @@ -1,7 +1,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 -Subject: [PATCH 5/8] Include jackson-dataformat-xml dependency. +Subject: Include jackson-dataformat-xml dependency. This allows us to use XmlLayout for Log4jV2. By including it here as a dependency we can make sure that we always have diff --git a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch index 9defabf62..ad09d484b 100644 --- a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch +++ b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch @@ -1,7 +1,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 -Subject: [PATCH 6/8] Stop building the tar.gz distribution. +Subject: Stop building the tar.gz distribution. All we do is build Druid tar and gzip it only to immediately uncompress it again. So, instead we just skip the compression step entirely. diff --git a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch index e04c92eb8..cba3d663c 100644 --- a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch +++ b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch @@ -1,7 +1,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Lukas Voetmand Date: Thu, 12 Dec 2024 17:59:17 +0100 -Subject: [PATCH 7/8] Update CycloneDX plugin +Subject: Update CycloneDX plugin --- pom.xml | 6 +++++- diff --git a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch index 794bcccb9..2a9aa20a6 100644 --- a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch +++ b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch @@ -1,7 +1,7 @@ From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 12 Dec 2024 17:59:17 +0100 -Subject: [PATCH 8/8] Fix CVE-2024-36114 +Subject: Fix CVE-2024-36114 See https://github.com/stackabletech/vulnerabilities/issues/834 diff --git a/patchable.nu b/patchable.nu index de48a4a88..292e83220 100755 --- a/patchable.nu +++ b/patchable.nu @@ -114,7 +114,7 @@ def "main export" [ log info "Deleting existing patches" rm ...(glob $"($patch_dir)/{*.patch,series}" | tee { print $in }) log info $"Exporting patches to ($patch_dir)" - git format-patch $config.base -o $patch_dir --base $config.base + git format-patch $config.base -o $patch_dir --base $config.base --keep-subject # Normally the patches include their own commit IDs, which will change for every for every reimport log info "Scrubbing commit ID from patches" sed -i "1s/From [0-9a-f]\\+ /From 0000000000000000000000000000000000000000 /" ...(glob $"($patch_dir)/*.patch") From 9b7e53f5cec8592a35eb68ad2c8005a4ee23ad34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:28:48 +0100 Subject: [PATCH 06/48] Add test patches --- .../stackable/patches/26.0.0/0009-asdf.patch | 25 +++++++++++++++++++ .../stackable/patches/26.0.0/0010-qwer.patch | 25 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 druid/stackable/patches/26.0.0/0009-asdf.patch create mode 100644 druid/stackable/patches/26.0.0/0010-qwer.patch diff --git a/druid/stackable/patches/26.0.0/0009-asdf.patch b/druid/stackable/patches/26.0.0/0009-asdf.patch new file mode 100644 index 000000000..8ef9e6149 --- /dev/null +++ b/druid/stackable/patches/26.0.0/0009-asdf.patch @@ -0,0 +1,25 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= +Date: Thu, 12 Dec 2024 18:26:17 +0100 +Subject: asdf + +--- + upload.sh | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/upload.sh b/upload.sh +index 2b49df1a49..9d421fa786 100755 +--- a/upload.sh ++++ b/upload.sh +@@ -18,6 +18,8 @@ + # Script to upload tarball of assembly build to static.druid.io for serving + # + ++echo a ++ + if [ $# -lt 1 ]; then + echo "Usage: $0 " >&2 + exit 2 +-- +2.47.1 + diff --git a/druid/stackable/patches/26.0.0/0010-qwer.patch b/druid/stackable/patches/26.0.0/0010-qwer.patch new file mode 100644 index 000000000..8993451f7 --- /dev/null +++ b/druid/stackable/patches/26.0.0/0010-qwer.patch @@ -0,0 +1,25 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= +Date: Thu, 12 Dec 2024 18:27:00 +0100 +Subject: qwer + +--- + upload.sh | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/upload.sh b/upload.sh +index 9d421fa786..5d910c9725 100755 +--- a/upload.sh ++++ b/upload.sh +@@ -25,6 +25,8 @@ if [ $# -lt 1 ]; then + exit 2 + fi + ++echo b ++ + VERSION=$1 + DRUID_TAR=druid-$VERSION-bin.tar.gz + MYSQL_TAR=mysql-metadata-storage-$VERSION.tar.gz +-- +2.47.1 + From 55de303cfea3e4a822d257bb6384876c4dd93b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:34:52 +0100 Subject: [PATCH 07/48] Rebase in test patch --- druid/stackable/patches/26.0.0/0009-asdf.patch | 4 ++-- druid/stackable/patches/26.0.0/0010-qwer.patch | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/druid/stackable/patches/26.0.0/0009-asdf.patch b/druid/stackable/patches/26.0.0/0009-asdf.patch index 8ef9e6149..30550dd39 100644 --- a/druid/stackable/patches/26.0.0/0009-asdf.patch +++ b/druid/stackable/patches/26.0.0/0009-asdf.patch @@ -8,14 +8,14 @@ Subject: asdf 1 file changed, 2 insertions(+) diff --git a/upload.sh b/upload.sh -index 2b49df1a49..9d421fa786 100755 +index 2b49df1a49..3927c205e3 100755 --- a/upload.sh +++ b/upload.sh @@ -18,6 +18,8 @@ # Script to upload tarball of assembly build to static.druid.io for serving # -+echo a ++echo a2 + if [ $# -lt 1 ]; then echo "Usage: $0 " >&2 diff --git a/druid/stackable/patches/26.0.0/0010-qwer.patch b/druid/stackable/patches/26.0.0/0010-qwer.patch index 8993451f7..19214c089 100644 --- a/druid/stackable/patches/26.0.0/0010-qwer.patch +++ b/druid/stackable/patches/26.0.0/0010-qwer.patch @@ -8,7 +8,7 @@ Subject: qwer 1 file changed, 2 insertions(+) diff --git a/upload.sh b/upload.sh -index 9d421fa786..5d910c9725 100755 +index 3927c205e3..493739e181 100755 --- a/upload.sh +++ b/upload.sh @@ -25,6 +25,8 @@ if [ $# -lt 1 ]; then From 3be52d4e3b15a903a5c926de0f01ba161d474393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:45:11 +0100 Subject: [PATCH 08/48] Add docs --- patchable.nu | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/patchable.nu b/patchable.nu index 292e83220..c10121a87 100755 --- a/patchable.nu +++ b/patchable.nu @@ -45,14 +45,29 @@ def product-version-config [product: string, version: string] { } def "main" [] { + print "Subcommands:" + print " ./patchable.nu checkout --help - Check out a product version from its patches" + print " ./patchable.nu export --help - Update a product's patches from its worktree" + print "" print "Usage:" - print " patchable checkout [--help]" + print " $ ./patchable.nu checkout druid 26.0.0" + print " $ enter druid/patchable-work/worktree/26.0.0/" + print " $ # do stuff" + print " $ git commit" + print " $ dexit" + print " $ ./patchable.nu export druid 26.0.0" + print " $ git status" } +# Check out a patched source tree from its upstream sources with patches applied. +# +# If the source tree already exists it will be overwritten. Old commits can be recovered from the git reflog. +# +# A separate source tree is maintained for each product. def "main checkout" [ - product: string - version: string - --force + product: string # The name of the product (example: druid) + version: string # The version of the product (example: 26.0.0) + --force # Overwrite existing checkouts somewhat more aggressively ] { let config = product-version-config $product $version let product_repo = (product-repo $product --upstream=$config.upstream) @@ -96,9 +111,10 @@ def "main checkout" [ } } +# Export the patches in the current source tree of a product. def "main export" [ - product: string - version: string + product: string # The name of the product (example: druid) + version: string # The version of the product (example: 26.0.0) ] { let config = product-version-config $product $version let product_repo = (product-repo $product --upstream=$config.upstream) From a2bc01a254e0757dbc24eecfaac24cf6333c727f Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Fri, 17 Jan 2025 16:07:11 +0100 Subject: [PATCH 09/48] fix: patchable worktree initialization (#977) --- patchable.nu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patchable.nu b/patchable.nu index c10121a87..41d860ab5 100755 --- a/patchable.nu +++ b/patchable.nu @@ -80,9 +80,9 @@ def "main checkout" [ # $GIT_DIR must be the worktree's .git dir, even if that is just an alias for the backing repo, since each worktree maintains its own index $env.GIT_DIR = $"($worktree_path)/.git" $env.GIT_WORK_TREE = $worktree_path - let worktree_git_dir = git rev-parse --git-dir - let worktree_rebase_progress_dir = $"($worktree_git_dir)/rebase-apply" if ($worktree_path | path exists) { + let worktree_git_dir = git rev-parse --git-dir + let worktree_rebase_progress_dir = $"($worktree_git_dir)/rebase-apply" log info "Worktree root already exists, resetting" if ($worktree_rebase_progress_dir | path exists) { if $force { From a0917565f472af5bbb1cbd7baf506042a4a96d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Tue, 4 Feb 2025 12:17:17 +0100 Subject: [PATCH 10/48] Start rewriting patchable in Rust --- .cargo/config.toml | 2 + .gitignore | 1 + Cargo.lock | 961 ++++++++++++++++++ Cargo.toml | 12 + ...traces-of-the-druid-ranger-extension.patch | 2 +- ...e-Prometheus-emitter-in-distribution.patch | 2 +- ...0003-Stop-building-unused-extensions.patch | 2 +- ...ndencies-that-have-a-new-patch-relea.patch | 2 +- ...de-jackson-dataformat-xml-dependency.patch | 2 +- ...top-building-the-tar.gz-distribution.patch | 2 +- .../26.0.0/0007-Update-CycloneDX-plugin.patch | 6 +- .../26.0.0/0008-Fix-CVE-2024-36114.patch | 4 +- .../stackable/patches/26.0.0/0009-asdf.patch | 2 +- .../stackable/patches/26.0.0/0010-qwer.patch | 2 +- patchable/Cargo.toml | 13 + patchable/src/main.rs | 268 +++++ 16 files changed, 1270 insertions(+), 13 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 patchable/Cargo.toml create mode 100644 patchable/src/main.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..b6cf3ff25 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +patchable = ["run", "--bin", "patchable", "--"] diff --git a/.gitignore b/.gitignore index 44e2c49f8..f9cdfdb08 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__/ *.py[cod] patchable-work/ +target/ diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..26419b9e9 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,961 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys", +] + +[[package]] +name = "bitflags" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" + +[[package]] +name = "cc" +version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4730490333d58093109dc02c23174c3f4d490998c3fed3cc8e82d57afedb9cf" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "4.5.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "git2" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fda788993cc341f69012feba8bf45c0ba4f3291fcc08e214b4d5a7332d88aff" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "libgit2-sys" +version = "0.18.0+1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1a117465e7e1597e8febea8bb0c410f1c7fb93b1e1cddf34363f8390367ffec" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "220e4f05ad4a218192533b300327f5150e809b54c4ec83b5a1d91833601811b9" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + +[[package]] +name = "log" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "patchable" +version = "0.1.0" +dependencies = [ + "clap", + "git2", + "regex", + "serde", + "toml", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "proc-macro2" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +dependencies = [ + "nu-ansi-term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "unicode-ident" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e49d2d35d3fad69b39b94139037ecfb4f359f08958b9c11e7315ce770462419" +dependencies = [ + "memchr", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..0eb78e919 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,12 @@ +[workspace] +members = ["patchable"] +resolver = "2" + +[workspace.dependencies] +clap = { version = "4.5.27", features = ["derive"] } +git2 = "0.20.0" +regex = "1.11.1" +serde = { version = "1.0.217", features = ["derive"] } +toml = "0.8.19" +tracing = "0.1.41" +tracing-subscriber = "0.3.19" diff --git a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch index 7b2f55b97..94b8ca49f 100644 --- a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch +++ b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch @@ -45,5 +45,5 @@ index 0c6294f5ed..a33c6bd521 100644 base-commit: 7cffb81a8e124d5f218f9af16ad685acf5e9c67c -- -2.47.1 +2.48.1 diff --git a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch index 919f93f94..9119baedf 100644 --- a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch +++ b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch @@ -65,5 +65,5 @@ index a6e72cf2c2..3ab13d5d11 100644 integration-test -- -2.47.1 +2.48.1 diff --git a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch index 28bd898cd..2babb1700 100644 --- a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch +++ b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch @@ -70,5 +70,5 @@ index a33c6bd521..f5001910e1 100644 ${repoOrgId} -- -2.47.1 +2.48.1 diff --git a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch index 585225df0..b2cf18f1b 100644 --- a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch +++ b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch @@ -133,5 +133,5 @@ index f5001910e1..2364f27dc4 100644 3.5.10 2.5.7 -- -2.47.1 +2.48.1 diff --git a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch index b62296f5d..1b99e5f78 100644 --- a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch +++ b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch @@ -28,5 +28,5 @@ index fdc6f1f548..9f18e614e9 100644 com.fasterxml.jackson.datatype jackson-datatype-joda -- -2.47.1 +2.48.1 diff --git a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch index ad09d484b..8d138b99a 100644 --- a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch +++ b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch @@ -23,5 +23,5 @@ index ff8e0d2fdd..f9daa49e21 100644 -- -2.47.1 +2.48.1 diff --git a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch index cba3d663c..3054ac18d 100644 --- a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch +++ b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch @@ -8,7 +8,7 @@ Subject: Update CycloneDX plugin 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml -index 2364f27dc4..208274a65a 100644 +index 2364f27dc4..c902899304 100644 --- a/pom.xml +++ b/pom.xml @@ -1533,7 +1533,11 @@ @@ -16,7 +16,7 @@ index 2364f27dc4..208274a65a 100644 org.cyclonedx cyclonedx-maven-plugin - 2.7.5 -+ 2.8.0 ++ 2.8.1 + + application + 1.5 @@ -25,5 +25,5 @@ index 2364f27dc4..208274a65a 100644 package -- -2.47.1 +2.48.1 diff --git a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch index 2a9aa20a6..3200cebf4 100644 --- a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch +++ b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch @@ -26,7 +26,7 @@ known workarounds for this issue. 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml -index 208274a65a..1d53874c41 100644 +index c902899304..6c24bdc0b2 100644 --- a/pom.xml +++ b/pom.xml @@ -233,6 +233,12 @@ @@ -43,5 +43,5 @@ index 208274a65a..1d53874c41 100644 commons-codec -- -2.47.1 +2.48.1 diff --git a/druid/stackable/patches/26.0.0/0009-asdf.patch b/druid/stackable/patches/26.0.0/0009-asdf.patch index 30550dd39..3bed3b17f 100644 --- a/druid/stackable/patches/26.0.0/0009-asdf.patch +++ b/druid/stackable/patches/26.0.0/0009-asdf.patch @@ -21,5 +21,5 @@ index 2b49df1a49..3927c205e3 100755 echo "Usage: $0 " >&2 exit 2 -- -2.47.1 +2.48.1 diff --git a/druid/stackable/patches/26.0.0/0010-qwer.patch b/druid/stackable/patches/26.0.0/0010-qwer.patch index 19214c089..e0a994c55 100644 --- a/druid/stackable/patches/26.0.0/0010-qwer.patch +++ b/druid/stackable/patches/26.0.0/0010-qwer.patch @@ -21,5 +21,5 @@ index 3927c205e3..493739e181 100755 DRUID_TAR=druid-$VERSION-bin.tar.gz MYSQL_TAR=mysql-metadata-storage-$VERSION.tar.gz -- -2.47.1 +2.48.1 diff --git a/patchable/Cargo.toml b/patchable/Cargo.toml new file mode 100644 index 000000000..070b8d0a1 --- /dev/null +++ b/patchable/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "patchable" +version = "0.1.0" +edition = "2021" + +[dependencies] +clap.workspace = true +git2.workspace = true +regex.workspace = true +serde.workspace = true +toml.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true diff --git a/patchable/src/main.rs b/patchable/src/main.rs new file mode 100644 index 000000000..f1fb62fd5 --- /dev/null +++ b/patchable/src/main.rs @@ -0,0 +1,268 @@ +use std::path::{Path, PathBuf}; + +use git2::{ + build::RepoBuilder, ObjectType, Repository, Signature, StatusOptions, WorktreeAddOptions, +}; +use regex::{Regex, RegexBuilder}; +use serde::Deserialize; + +#[derive(clap::Parser)] +struct ProductVersion { + product: String, + version: String, +} + +impl ProductVersion { + fn root(&self, images_root: &Path) -> PathBuf { + images_root.join(&self.product) + } + + fn patch_dir(&self, images_root: &Path) -> PathBuf { + self.root(images_root) + .join("stackable/patches") + .join(&self.version) + } + + fn config_path(&self, images_root: &Path) -> PathBuf { + self.patch_dir(images_root).join("patchable.toml") + } + + fn work_root(&self, images_root: &Path) -> PathBuf { + self.root(images_root).join("patchable-work") + } + + fn repo(&self, images_root: &Path) -> PathBuf { + self.work_root(images_root).join("product-repo") + } + + fn worktree_root(&self, images_root: &Path) -> PathBuf { + self.work_root(images_root) + .join("worktree") + .join(&self.version) + } + + fn worktree_branch(&self) -> String { + format!("patchable/{}", self.version) + } +} + +#[derive(Deserialize)] +struct ProductVersionConfig { + upstream: String, + base: String, +} + +#[derive(clap::Parser)] +struct Opts { + #[clap(subcommand)] + cmd: Cmd, +} + +#[derive(clap::Parser)] +enum Cmd { + Checkout { + #[clap(flatten)] + pv: ProductVersion, + }, + Export { + #[clap(flatten)] + pv: ProductVersion, + }, +} + +fn main() { + tracing_subscriber::fmt().init(); + let opts = ::parse(); + let images_repo = Repository::discover(".").unwrap(); + let images_repo_root = images_repo.workdir().unwrap(); + match opts.cmd { + Cmd::Checkout { pv } => { + let config = toml::from_str::( + &std::fs::read_to_string(pv.config_path(images_repo_root)).unwrap(), + ) + .unwrap(); + let product_repo_root = pv.repo(images_repo_root); + let product_repo = match Repository::open(&product_repo_root) { + Ok(repo) => repo, + Err(_) => RepoBuilder::new() + .bare(true) + .clone(&config.upstream, &product_repo_root) + .unwrap(), + }; + let product_worktree_root = pv.worktree_root(images_repo_root); + let mut product_version_repo = match Repository::open(&product_worktree_root) { + Ok(wt) => wt, + Err(err) => { + tracing::info!( + error = &err as &dyn std::error::Error, + "worktree not found, creating" + ); + Repository::open_from_worktree( + &product_repo + .worktree( + &pv.version, + &product_worktree_root, + Some(&WorktreeAddOptions::new()), + ) + .unwrap(), + ) + .unwrap() + } + }; + product_version_repo.cleanup_state().unwrap(); + + let stash = if product_version_repo + .statuses(Some(StatusOptions::new().include_unmodified(false))) + .unwrap() + .is_empty() + { + None + } else { + Some( + product_version_repo + .stash_save( + &Signature::now("Patchable", "noreply+patchable@stackable.tech") + .unwrap(), + "Existing work before checking out ", + None, + ) + .unwrap(), + ) + }; + + product_version_repo + .set_head_detached( + product_version_repo + .head() + .unwrap() + .peel_to_commit() + .unwrap() + .id(), + ) + .unwrap(); + { + let branch = product_version_repo + .branch( + &pv.worktree_branch(), + &product_version_repo + .revparse_single(&config.base) + .unwrap() + .as_commit() + .unwrap(), + true, + ) + .unwrap() + .into_reference(); + product_version_repo + .checkout_tree(&branch.peel(ObjectType::Commit).unwrap(), None) + .unwrap(); + product_version_repo + .set_head_bytes(branch.name_bytes()) + .unwrap(); + } + + let patch_dir = pv.patch_dir(images_repo_root); + let series_file = patch_dir.join("series"); + let mut apply_cmd = raw_git_cmd(&product_version_repo); + if series_file.exists() { + apply_cmd + .arg("am") + .arg(series_file) + .args(["--patch-format", "stgit-series"]); + } else { + let mut patch_files = patch_dir + .read_dir() + .unwrap() + .map(|x| x.unwrap().path()) + .filter(|x| x.extension().is_some_and(|x| x == "patch")) + .collect::>(); + patch_files.sort(); + apply_cmd.arg("am").args(patch_files); + } + if !apply_cmd.spawn().unwrap().wait().unwrap().success() { + panic!("failed to apply patches"); + } + + if let Some(stash) = stash { + let mut stash_index = None; + product_version_repo + .stash_foreach(|i, _, oid| { + if oid == &stash { + stash_index = Some(i); + true + } else { + false + } + }) + .unwrap(); + product_version_repo + .stash_pop(stash_index.unwrap(), None) + .unwrap(); + } + } + Cmd::Export { pv } => { + let config = toml::from_str::( + &std::fs::read_to_string(pv.config_path(images_repo_root)).unwrap(), + ) + .unwrap(); + + let patch_dir = pv.patch_dir(images_repo_root); + for entry in patch_dir.read_dir().unwrap() { + let path = entry.unwrap().path(); + if path.file_name().is_some_and(|x| x == "series") + || path.extension().is_some_and(|x| x == "patch") + { + std::fs::remove_file(path).unwrap(); + } + } + + let product_version_repo = + Repository::open(pv.worktree_root(images_repo_root)).unwrap(); + if !raw_git_cmd(&product_version_repo) + .arg("format-patch") + .arg(&config.base) + .arg("-o") + .arg(&patch_dir) + .arg("--base") + .arg(&config.base) + .arg("--keep-subject") + .spawn() + .unwrap() + .wait() + .unwrap() + .success() + { + panic!("failed to format patches"); + } + + let regex_line = RegexBuilder::new("^.*$").multi_line(true).build().unwrap(); + let regex_from = Regex::new("^From [0-9a-f]+ ").unwrap(); + + for entry in patch_dir.read_dir().unwrap() { + let path = entry.unwrap().path(); + if path.extension().is_some_and(|x| x == "patch") { + let mut patch_file = std::fs::read_to_string(&path).unwrap(); + let line_1 = regex_line.find(&patch_file).unwrap(); + let from = regex_from + .find_at(&patch_file[..line_1.end()], line_1.start()) + .unwrap(); + patch_file.replace_range( + from.range(), + "From 0000000000000000000000000000000000000000 ", + ); + std::fs::write(path, patch_file).unwrap(); + } + } + } + } +} + +/// Runs a raw git command in the environment of a Git repository. +/// +/// Used for functionality that is not currently implemented by libgit2/gix. +fn raw_git_cmd(repo: &Repository) -> std::process::Command { + let mut cmd = std::process::Command::new("git"); + cmd.env("GIT_DIR", repo.path()) + .env("GIT_WORK_TREE", repo.workdir().unwrap()); + cmd +} From 619852bb704acf31324f4285c64710db9e3e2b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 5 Feb 2025 12:51:40 +0100 Subject: [PATCH 11/48] Logging --- patchable/src/main.rs | 214 ++++++++++++++++++++++++++++++------------ 1 file changed, 153 insertions(+), 61 deletions(-) diff --git a/patchable/src/main.rs b/patchable/src/main.rs index f1fb62fd5..dd84baca2 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -12,44 +12,56 @@ struct ProductVersion { version: String, } -impl ProductVersion { - fn root(&self, images_root: &Path) -> PathBuf { - images_root.join(&self.product) +#[derive(Deserialize)] +struct ProductVersionConfig { + upstream: String, + base: String, +} + +struct ProductVersionContext<'a> { + pv: ProductVersion, + images_repo_root: &'a Path, +} + +impl ProductVersionContext<'_> { + fn load_config(&self) -> ProductVersionConfig { + tracing::info!( + config.path = %self.config_path().display(), + "loading config" + ); + toml::from_str::( + &std::fs::read_to_string(self.config_path()).unwrap(), + ) + .unwrap() } - fn patch_dir(&self, images_root: &Path) -> PathBuf { - self.root(images_root) - .join("stackable/patches") - .join(&self.version) + fn root(&self) -> PathBuf { + self.images_repo_root.join(&self.pv.product) } - fn config_path(&self, images_root: &Path) -> PathBuf { - self.patch_dir(images_root).join("patchable.toml") + fn patch_dir(&self) -> PathBuf { + self.root().join("stackable/patches").join(&self.pv.version) } - fn work_root(&self, images_root: &Path) -> PathBuf { - self.root(images_root).join("patchable-work") + fn config_path(&self) -> PathBuf { + self.patch_dir().join("patchable.toml") } - fn repo(&self, images_root: &Path) -> PathBuf { - self.work_root(images_root).join("product-repo") + fn work_root(&self) -> PathBuf { + self.root().join("patchable-work") } - fn worktree_root(&self, images_root: &Path) -> PathBuf { - self.work_root(images_root) - .join("worktree") - .join(&self.version) + fn repo(&self) -> PathBuf { + self.work_root().join("product-repo") } - fn worktree_branch(&self) -> String { - format!("patchable/{}", self.version) + fn worktree_root(&self) -> PathBuf { + self.work_root().join("worktree").join(&self.pv.version) } -} -#[derive(Deserialize)] -struct ProductVersionConfig { - upstream: String, - base: String, + fn worktree_branch(&self) -> String { + format!("patchable/{}", self.pv.version) + } } #[derive(clap::Parser)] @@ -77,30 +89,53 @@ fn main() { let images_repo_root = images_repo.workdir().unwrap(); match opts.cmd { Cmd::Checkout { pv } => { - let config = toml::from_str::( - &std::fs::read_to_string(pv.config_path(images_repo_root)).unwrap(), - ) - .unwrap(); - let product_repo_root = pv.repo(images_repo_root); + let ctx = ProductVersionContext { + pv, + images_repo_root, + }; + let config = ctx.load_config(); + let product_repo_root = ctx.repo(); let product_repo = match Repository::open(&product_repo_root) { - Ok(repo) => repo, - Err(_) => RepoBuilder::new() - .bare(true) - .clone(&config.upstream, &product_repo_root) - .unwrap(), + Ok(repo) => { + tracing::info!( + product.repository = %product_repo_root.display(), + "product repository found, reusing" + ); + repo + } + Err(err) => { + tracing::info!( + error = &err as &dyn std::error::Error, + product.repository = %product_repo_root.display(), + product.upstream = config.upstream, + "product repository not found, cloning from upstream" + ); + RepoBuilder::new() + .bare(true) + .clone(&config.upstream, &product_repo_root) + .unwrap() + } }; - let product_worktree_root = pv.worktree_root(images_repo_root); + let product_worktree_root = ctx.worktree_root(); let mut product_version_repo = match Repository::open(&product_worktree_root) { - Ok(wt) => wt, + Ok(wt) => { + tracing::info!( + worktree.root = %product_worktree_root.display(), + "worktree found, resetting and reusing" + ); + wt + } Err(err) => { tracing::info!( error = &err as &dyn std::error::Error, + worktree.root = %product_worktree_root.display(), + product.repository = %product_repo_root.display(), "worktree not found, creating" ); Repository::open_from_worktree( &product_repo .worktree( - &pv.version, + &ctx.pv.version, &product_worktree_root, Some(&WorktreeAddOptions::new()), ) @@ -109,6 +144,8 @@ fn main() { .unwrap() } }; + + tracing::info!("cleaning up existing rebase state"); product_version_repo.cleanup_state().unwrap(); let stash = if product_version_repo @@ -116,20 +153,28 @@ fn main() { .unwrap() .is_empty() { + tracing::info!("worktree is clean, no need to stash"); None } else { - Some( - product_version_repo - .stash_save( - &Signature::now("Patchable", "noreply+patchable@stackable.tech") - .unwrap(), - "Existing work before checking out ", - None, - ) - .unwrap(), - ) + tracing::warn!("worktree is dirty, stashing changes!"); + let stash = product_version_repo + .stash_save( + &Signature::now("Patchable", "noreply+patchable@stackable.tech").unwrap(), + "Existing work before checking out ", + None, + ) + .unwrap(); + tracing::info!(%stash, "created stash"); + Some(stash) }; + let worktree_branch = ctx.worktree_branch(); + tracing::info!( + worktree.branch = worktree_branch, + worktree.branch.base = config.base, + "checking out base commit into branch" + ); + // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime product_version_repo .set_head_detached( product_version_repo @@ -143,8 +188,8 @@ fn main() { { let branch = product_version_repo .branch( - &pv.worktree_branch(), - &product_version_repo + &worktree_branch, + product_version_repo .revparse_single(&config.base) .unwrap() .as_commit() @@ -161,15 +206,27 @@ fn main() { .unwrap(); } - let patch_dir = pv.patch_dir(images_repo_root); + let patch_dir = ctx.patch_dir(); + tracing::info!( + patch.dir = %patch_dir.display(), + "applying patches" + ); let series_file = patch_dir.join("series"); let mut apply_cmd = raw_git_cmd(&product_version_repo); if series_file.exists() { + tracing::info!( + patch.series = %series_file.display(), + "series file found, treating as stgit series" + ); apply_cmd .arg("am") .arg(series_file) .args(["--patch-format", "stgit-series"]); } else { + tracing::info!( + patch.series = %series_file.display(), + "series file not found, treating as git mailbox" + ); let mut patch_files = patch_dir .read_dir() .unwrap() @@ -195,20 +252,44 @@ fn main() { } }) .unwrap(); - product_version_repo - .stash_pop(stash_index.unwrap(), None) - .unwrap(); + let stash_index = stash_index.unwrap(); + tracing::info!( + stash = %format_args!("stash@{{{stash_index}}}"), + "restoring stash" + ); + product_version_repo.stash_pop(stash_index, None).unwrap(); } + + tracing::info!( + worktree.root = %product_worktree_root.display(), + "worktree is ready!" + ); } Cmd::Export { pv } => { - let config = toml::from_str::( - &std::fs::read_to_string(pv.config_path(images_repo_root)).unwrap(), - ) - .unwrap(); + let ctx = ProductVersionContext { + pv, + images_repo_root, + }; + let config = ctx.load_config(); + + let product_worktree_root = ctx.worktree_root(); + tracing::info!( + worktree.root = %product_worktree_root.display(), + "opening product worktree" + ); + let product_version_repo = Repository::open(&product_worktree_root).unwrap(); - let patch_dir = pv.patch_dir(images_repo_root); + let patch_dir = ctx.patch_dir(); + tracing::info!( + patch.dir = %patch_dir.display(), + "deleting existing patch files" + ); + // git format-patch is happy to overwrite existing files, + // but we also want to delete removed (or renamed) patch files. for entry in patch_dir.read_dir().unwrap() { let path = entry.unwrap().path(); + // git format-patch emits the mailbox format, not stgit(/quilt), + // so also remove markers that make it look like that if path.file_name().is_some_and(|x| x == "series") || path.extension().is_some_and(|x| x == "patch") { @@ -216,8 +297,12 @@ fn main() { } } - let product_version_repo = - Repository::open(pv.worktree_root(images_repo_root)).unwrap(); + tracing::info!( + patch.dir = %patch_dir.display(), + worktree.root = %product_worktree_root.display(), + worktree.base = config.base, + "exporting commits since base" + ); if !raw_git_cmd(&product_version_repo) .arg("format-patch") .arg(&config.base) @@ -235,9 +320,11 @@ fn main() { panic!("failed to format patches"); } + // Normally the patches include their own commit IDs, which will change for every for every re-checkout + // checkout doesn't actually care about this value, so we can just replace it with a deterministic dummy + tracing::info!("scrubbing commit ID from exported patches"); let regex_line = RegexBuilder::new("^.*$").multi_line(true).build().unwrap(); let regex_from = Regex::new("^From [0-9a-f]+ ").unwrap(); - for entry in patch_dir.read_dir().unwrap() { let path = entry.unwrap().path(); if path.extension().is_some_and(|x| x == "patch") { @@ -253,6 +340,11 @@ fn main() { std::fs::write(path, patch_file).unwrap(); } } + + tracing::info!( + patch.dir = %patch_dir.display(), + "worktree is exported!" + ); } } } From 034b9118622b13f3fd9a8d03bc8c69756c3c988b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 5 Feb 2025 13:43:39 +0100 Subject: [PATCH 12/48] Fix broken initial checkout --- patchable/src/main.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/patchable/src/main.rs b/patchable/src/main.rs index dd84baca2..83d335b31 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -1,4 +1,7 @@ -use std::path::{Path, PathBuf}; +use std::{ + path::{Path, PathBuf}, + process::Stdio, +}; use git2::{ build::RepoBuilder, ObjectType, Repository, Signature, StatusOptions, WorktreeAddOptions, @@ -60,7 +63,7 @@ impl ProductVersionContext<'_> { } fn worktree_branch(&self) -> String { - format!("patchable/{}", self.pv.version) + format!("patchable-{}", self.pv.version) } } @@ -117,6 +120,7 @@ fn main() { } }; let product_worktree_root = ctx.worktree_root(); + let worktree_branch = ctx.worktree_branch(); let mut product_version_repo = match Repository::open(&product_worktree_root) { Ok(wt) => { tracing::info!( @@ -132,10 +136,11 @@ fn main() { product.repository = %product_repo_root.display(), "worktree not found, creating" ); + std::fs::create_dir_all(product_worktree_root.parent().unwrap()).unwrap(); Repository::open_from_worktree( &product_repo .worktree( - &ctx.pv.version, + &worktree_branch, &product_worktree_root, Some(&WorktreeAddOptions::new()), ) @@ -168,7 +173,6 @@ fn main() { Some(stash) }; - let worktree_branch = ctx.worktree_branch(); tracing::info!( worktree.branch = worktree_branch, worktree.branch.base = config.base, @@ -236,7 +240,7 @@ fn main() { patch_files.sort(); apply_cmd.arg("am").args(patch_files); } - if !apply_cmd.spawn().unwrap().wait().unwrap().success() { + if !apply_cmd.status().unwrap().success() { panic!("failed to apply patches"); } @@ -311,9 +315,7 @@ fn main() { .arg("--base") .arg(&config.base) .arg("--keep-subject") - .spawn() - .unwrap() - .wait() + .status() .unwrap() .success() { From a99104b18c820df7dede4d9e55ea4b666ddeab6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 5 Feb 2025 14:41:46 +0100 Subject: [PATCH 13/48] Fetch commits individually from upstream instead of cloning --- patchable/src/main.rs | 71 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/patchable/src/main.rs b/patchable/src/main.rs index 83d335b31..a30e234f2 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -1,10 +1,8 @@ -use std::{ - path::{Path, PathBuf}, - process::Stdio, -}; +use std::path::{Path, PathBuf}; use git2::{ - build::RepoBuilder, ObjectType, Repository, Signature, StatusOptions, WorktreeAddOptions, + FetchOptions, ObjectType, Repository, RepositoryInitOptions, Signature, StatusOptions, + WorktreeAddOptions, }; use regex::{Regex, RegexBuilder}; use serde::Deserialize; @@ -63,7 +61,7 @@ impl ProductVersionContext<'_> { } fn worktree_branch(&self) -> String { - format!("patchable-{}", self.pv.version) + format!("patchable/{}", self.pv.version) } } @@ -111,14 +109,49 @@ fn main() { error = &err as &dyn std::error::Error, product.repository = %product_repo_root.display(), product.upstream = config.upstream, - "product repository not found, cloning from upstream" + "product repository not found, initializing" ); - RepoBuilder::new() - .bare(true) - .clone(&config.upstream, &product_repo_root) - .unwrap() + let repo = Repository::init_opts( + &product_repo_root, + RepositoryInitOptions::new() + .bare(true) + .initial_head("patchable-dummy") + .external_template(false), + ) + .unwrap(); + + repo } }; + + match product_repo.revparse_single(&config.base) { + Ok(_) => tracing::info!( + worktree.branch.base = config.base, + "base commit exists, reusing" + ), + Err(err) => { + tracing::info!( + error = &err as &dyn std::error::Error, + worktree.branch.base = config.base, + product.upstream = config.upstream, + "base commit not found, fetching from upstream" + ); + product_repo + .remote_anonymous(&config.upstream) + .unwrap() + .fetch( + &[&config.base], + Some( + FetchOptions::new() + // TODO: could be 1, CLI option maybe? + .depth(0), + ), + None, + ) + .unwrap(); + } + } + let product_worktree_root = ctx.worktree_root(); let worktree_branch = ctx.worktree_branch(); let mut product_version_repo = match Repository::open(&product_worktree_root) { @@ -137,12 +170,24 @@ fn main() { "worktree not found, creating" ); std::fs::create_dir_all(product_worktree_root.parent().unwrap()).unwrap(); + let worktree_ref = product_repo + .branch( + &worktree_branch, + &product_repo + .revparse_single(&config.base) + .unwrap() + .peel_to_commit() + .unwrap(), + true, + ) + .unwrap() + .into_reference(); Repository::open_from_worktree( &product_repo .worktree( - &worktree_branch, + &ctx.pv.version, &product_worktree_root, - Some(&WorktreeAddOptions::new()), + Some(WorktreeAddOptions::new().reference(Some(&worktree_ref))), ) .unwrap(), ) From d11241b86a58bbaacec97c9fa631b0dceba0a288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 6 Feb 2025 13:09:36 +0100 Subject: [PATCH 14/48] Make commit IDs deterministic --- ...traces-of-the-druid-ranger-extension.patch | 2 +- ...e-Prometheus-emitter-in-distribution.patch | 2 +- ...0003-Stop-building-unused-extensions.patch | 2 +- ...ndencies-that-have-a-new-patch-relea.patch | 2 +- ...de-jackson-dataformat-xml-dependency.patch | 2 +- ...top-building-the-tar.gz-distribution.patch | 2 +- .../26.0.0/0007-Update-CycloneDX-plugin.patch | 2 +- .../26.0.0/0008-Fix-CVE-2024-36114.patch | 2 +- .../stackable/patches/26.0.0/0009-asdf.patch | 2 +- .../stackable/patches/26.0.0/0010-qwer.patch | 2 +- patchable/src/main.rs | 23 ++----------------- 11 files changed, 12 insertions(+), 31 deletions(-) diff --git a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch index 94b8ca49f..1249d7785 100644 --- a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch +++ b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From c22698f41d83103d549e60f9dfa08539531d9eb6 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Removes all traces of the druid ranger extension diff --git a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch index 9119baedf..333fa38cb 100644 --- a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch +++ b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From dc8b5d9b733ab542b7af038f19361d39be5cba60 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Include Prometheus emitter in distribution diff --git a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch index 2babb1700..9b2cbb0ee 100644 --- a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch +++ b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From fd3cf92c1be84284a1a892ba039b6d542c21bc1e Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Stop building unused extensions. diff --git a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch index b2cf18f1b..be5fdc113 100644 --- a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch +++ b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From 6b5af7db810d38f46806df32f7d2db27ebd79b05 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Updates all dependencies that have a new patch release available. diff --git a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch index 1b99e5f78..948f9a895 100644 --- a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch +++ b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From 0b4fd5532600d94e032f15b4bd9896ed66256b51 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Include jackson-dataformat-xml dependency. diff --git a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch index 8d138b99a..a99aaabea 100644 --- a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch +++ b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From ad43c6cc399d73017ae1db8877bda78f391ebf1b Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Stop building the tar.gz distribution. diff --git a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch index 3054ac18d..f247fb218 100644 --- a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch +++ b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From 89cf59dfab4e82cce4827fd6b08fd3506aada9d2 Mon Sep 17 00:00:00 2001 From: Lukas Voetmand Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Update CycloneDX plugin diff --git a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch index 3200cebf4..401a19c44 100644 --- a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch +++ b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From ab5b46c818010d58e137fdecdeabe8e7223d6667 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Fix CVE-2024-36114 diff --git a/druid/stackable/patches/26.0.0/0009-asdf.patch b/druid/stackable/patches/26.0.0/0009-asdf.patch index 3bed3b17f..4f0dcb7bc 100644 --- a/druid/stackable/patches/26.0.0/0009-asdf.patch +++ b/druid/stackable/patches/26.0.0/0009-asdf.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From a6e0a2db29011abe455306b9410021ff3b153b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:26:17 +0100 Subject: asdf diff --git a/druid/stackable/patches/26.0.0/0010-qwer.patch b/druid/stackable/patches/26.0.0/0010-qwer.patch index e0a994c55..d6ec7cb22 100644 --- a/druid/stackable/patches/26.0.0/0010-qwer.patch +++ b/druid/stackable/patches/26.0.0/0010-qwer.patch @@ -1,4 +1,4 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From 7865cd4163133ed3dcfe2be474dcb91484e19324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:27:00 +0100 Subject: qwer diff --git a/patchable/src/main.rs b/patchable/src/main.rs index a30e234f2..5ac10b4f2 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -285,6 +285,8 @@ fn main() { patch_files.sort(); apply_cmd.arg("am").args(patch_files); } + // Make commit hashes deterministic by removing variables that depend on the local environment + apply_cmd.args(["--committer-date-is-author-date"]); if !apply_cmd.status().unwrap().success() { panic!("failed to apply patches"); } @@ -367,27 +369,6 @@ fn main() { panic!("failed to format patches"); } - // Normally the patches include their own commit IDs, which will change for every for every re-checkout - // checkout doesn't actually care about this value, so we can just replace it with a deterministic dummy - tracing::info!("scrubbing commit ID from exported patches"); - let regex_line = RegexBuilder::new("^.*$").multi_line(true).build().unwrap(); - let regex_from = Regex::new("^From [0-9a-f]+ ").unwrap(); - for entry in patch_dir.read_dir().unwrap() { - let path = entry.unwrap().path(); - if path.extension().is_some_and(|x| x == "patch") { - let mut patch_file = std::fs::read_to_string(&path).unwrap(); - let line_1 = regex_line.find(&patch_file).unwrap(); - let from = regex_from - .find_at(&patch_file[..line_1.end()], line_1.start()) - .unwrap(); - patch_file.replace_range( - from.range(), - "From 0000000000000000000000000000000000000000 ", - ); - std::fs::write(path, patch_file).unwrap(); - } - } - tracing::info!( patch.dir = %patch_dir.display(), "worktree is exported!" From 3320a066f22ebb43613f4d6465998eed2e805567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 6 Feb 2025 14:42:23 +0100 Subject: [PATCH 15/48] Reimplement git-am for better determinism --- Cargo.lock | 161 +++++++++--- Cargo.toml | 3 +- ...traces-of-the-druid-ranger-extension.patch | 2 +- ...e-Prometheus-emitter-in-distribution.patch | 2 +- ...0003-Stop-building-unused-extensions.patch | 2 +- ...ndencies-that-have-a-new-patch-relea.patch | 2 +- ...de-jackson-dataformat-xml-dependency.patch | 2 +- ...top-building-the-tar.gz-distribution.patch | 2 +- .../26.0.0/0007-Update-CycloneDX-plugin.patch | 2 +- .../26.0.0/0008-Fix-CVE-2024-36114.patch | 2 +- .../stackable/patches/26.0.0/0009-asdf.patch | 2 +- .../stackable/patches/26.0.0/0010-qwer.patch | 2 +- patchable/Cargo.toml | 3 +- patchable/src/main.rs | 229 +++++++++++------- 14 files changed, 288 insertions(+), 128 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26419b9e9..52b019db6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - [[package]] name = "anstream" version = "0.6.18" @@ -130,6 +121,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -147,6 +147,22 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -156,6 +172,18 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi", + "windows-targets", +] + [[package]] name = "git2" version = "0.20.0" @@ -399,6 +427,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "litemap" version = "0.7.4" @@ -427,6 +461,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "once_cell" version = "1.20.2" @@ -463,8 +503,9 @@ version = "0.1.0" dependencies = [ "clap", "git2", - "regex", "serde", + "tempfile", + "time", "toml", "tracing", "tracing-subscriber", @@ -488,6 +529,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro2" version = "1.0.93" @@ -507,34 +554,18 @@ dependencies = [ ] [[package]] -name = "regex" -version = "1.11.1" +name = "rustix" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", ] -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - [[package]] name = "serde" version = "1.0.217" @@ -619,6 +650,20 @@ dependencies = [ "syn", ] +[[package]] +name = "tempfile" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" +dependencies = [ + "cfg-if", + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thread_local" version = "1.1.8" @@ -629,6 +674,36 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -777,6 +852,15 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "winapi" version = "0.3.9" @@ -881,6 +965,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 0eb78e919..9f125fa62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,8 +5,9 @@ resolver = "2" [workspace.dependencies] clap = { version = "4.5.27", features = ["derive"] } git2 = "0.20.0" -regex = "1.11.1" serde = { version = "1.0.217", features = ["derive"] } +tempfile = "3.16.0" +time = { version = "0.3.37", features = ["parsing"] } toml = "0.8.19" tracing = "0.1.41" tracing-subscriber = "0.3.19" diff --git a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch index 1249d7785..87f022916 100644 --- a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch +++ b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch @@ -1,4 +1,4 @@ -From c22698f41d83103d549e60f9dfa08539531d9eb6 Mon Sep 17 00:00:00 2001 +From a8bec93ee6d0a4364676333168229aa0ec56657e Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Removes all traces of the druid ranger extension diff --git a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch index 333fa38cb..153955a4a 100644 --- a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch +++ b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch @@ -1,4 +1,4 @@ -From dc8b5d9b733ab542b7af038f19361d39be5cba60 Mon Sep 17 00:00:00 2001 +From c19288cd84492d76f924152f2d4f0d0fc0499ed6 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Include Prometheus emitter in distribution diff --git a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch index 9b2cbb0ee..8c93ad9a9 100644 --- a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch +++ b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch @@ -1,4 +1,4 @@ -From fd3cf92c1be84284a1a892ba039b6d542c21bc1e Mon Sep 17 00:00:00 2001 +From ca72868ea21177edd5e932d1915f7ad7f04de069 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Stop building unused extensions. diff --git a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch index be5fdc113..72c889c97 100644 --- a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch +++ b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch @@ -1,4 +1,4 @@ -From 6b5af7db810d38f46806df32f7d2db27ebd79b05 Mon Sep 17 00:00:00 2001 +From 74a9e778a1b06904566dd3c0ed7a4814459efdcb Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Updates all dependencies that have a new patch release available. diff --git a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch index 948f9a895..2fbbc586d 100644 --- a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch +++ b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch @@ -1,4 +1,4 @@ -From 0b4fd5532600d94e032f15b4bd9896ed66256b51 Mon Sep 17 00:00:00 2001 +From 4276496b4df4c5e40df3b27bf9fc989b4ee9259b Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Include jackson-dataformat-xml dependency. diff --git a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch index a99aaabea..acc3e500b 100644 --- a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch +++ b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch @@ -1,4 +1,4 @@ -From ad43c6cc399d73017ae1db8877bda78f391ebf1b Mon Sep 17 00:00:00 2001 +From 92dcd86aec69c174a590d6854b428af93864c2c4 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Stop building the tar.gz distribution. diff --git a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch index f247fb218..5fd7179d0 100644 --- a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch +++ b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch @@ -1,4 +1,4 @@ -From 89cf59dfab4e82cce4827fd6b08fd3506aada9d2 Mon Sep 17 00:00:00 2001 +From f9520996498906692fd56bc7ffc35f385d14ce64 Mon Sep 17 00:00:00 2001 From: Lukas Voetmand Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Update CycloneDX plugin diff --git a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch index 401a19c44..f7efbc9ea 100644 --- a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch +++ b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch @@ -1,4 +1,4 @@ -From ab5b46c818010d58e137fdecdeabe8e7223d6667 Mon Sep 17 00:00:00 2001 +From 377483cdea7a88ddf17ea364b265e58e721b985a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Fix CVE-2024-36114 diff --git a/druid/stackable/patches/26.0.0/0009-asdf.patch b/druid/stackable/patches/26.0.0/0009-asdf.patch index 4f0dcb7bc..04dedb8b5 100644 --- a/druid/stackable/patches/26.0.0/0009-asdf.patch +++ b/druid/stackable/patches/26.0.0/0009-asdf.patch @@ -1,4 +1,4 @@ -From a6e0a2db29011abe455306b9410021ff3b153b74 Mon Sep 17 00:00:00 2001 +From 4d87008741b2b6356147b4b6ef648f49971f3cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:26:17 +0100 Subject: asdf diff --git a/druid/stackable/patches/26.0.0/0010-qwer.patch b/druid/stackable/patches/26.0.0/0010-qwer.patch index d6ec7cb22..7ee4f5c49 100644 --- a/druid/stackable/patches/26.0.0/0010-qwer.patch +++ b/druid/stackable/patches/26.0.0/0010-qwer.patch @@ -1,4 +1,4 @@ -From 7865cd4163133ed3dcfe2be474dcb91484e19324 Mon Sep 17 00:00:00 2001 +From f793faeaa7182e7e997bd7571ac1e7e508b09077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:27:00 +0100 Subject: qwer diff --git a/patchable/Cargo.toml b/patchable/Cargo.toml index 070b8d0a1..9874acb1a 100644 --- a/patchable/Cargo.toml +++ b/patchable/Cargo.toml @@ -6,8 +6,9 @@ edition = "2021" [dependencies] clap.workspace = true git2.workspace = true -regex.workspace = true serde.workspace = true +tempfile.workspace = true +time.workspace = true toml.workspace = true tracing.workspace = true tracing-subscriber.workspace = true diff --git a/patchable/src/main.rs b/patchable/src/main.rs index 5ac10b4f2..f45012eba 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -1,11 +1,17 @@ -use std::path::{Path, PathBuf}; +use core::str; +use std::{ + fs::File, + path::{Path, PathBuf}, + process::Stdio, +}; use git2::{ - FetchOptions, ObjectType, Repository, RepositoryInitOptions, Signature, StatusOptions, + Diff, FetchOptions, ObjectType, Repository, RepositoryInitOptions, Signature, WorktreeAddOptions, }; -use regex::{Regex, RegexBuilder}; use serde::Deserialize; +use tempfile::tempdir; +use time::{format_description::well_known::Rfc2822, OffsetDateTime}; #[derive(clap::Parser)] struct ProductVersion { @@ -108,7 +114,6 @@ fn main() { tracing::info!( error = &err as &dyn std::error::Error, product.repository = %product_repo_root.display(), - product.upstream = config.upstream, "product repository not found, initializing" ); let repo = Repository::init_opts( @@ -154,7 +159,7 @@ fn main() { let product_worktree_root = ctx.worktree_root(); let worktree_branch = ctx.worktree_branch(); - let mut product_version_repo = match Repository::open(&product_worktree_root) { + let product_version_repo = match Repository::open(&product_worktree_root) { Ok(wt) => { tracing::info!( worktree.root = %product_worktree_root.display(), @@ -198,30 +203,150 @@ fn main() { tracing::info!("cleaning up existing rebase state"); product_version_repo.cleanup_state().unwrap(); - let stash = if product_version_repo - .statuses(Some(StatusOptions::new().include_unmodified(false))) - .unwrap() - .is_empty() - { - tracing::info!("worktree is clean, no need to stash"); - None + let patch_dir = ctx.patch_dir(); + tracing::info!( + patch.dir = %patch_dir.display(), + "applying patches" + ); + // This effectively reimplements git-am, but lets us: + // 1. Fake the committer information to match author (to ensure that we get deterministic committer IDs across machines) + // 2. Avoid touching the worktree until all patches have been applied + // (letting us keep any dirty files in the worktree that don't conflict with the final switcheroo, + // even if those files are modified by some of the patches) + let mailsplit_dir = tempdir().unwrap(); + let series_file = patch_dir.join("series"); + let patch_files = if series_file.exists() { + tracing::info!( + patch.series = %series_file.display(), + "series file found, treating as stgit series" + ); + std::fs::read_to_string(series_file) + .unwrap() + .lines() + .map(|file_name| patch_dir.join(file_name)) + .collect::>() } else { - tracing::warn!("worktree is dirty, stashing changes!"); - let stash = product_version_repo - .stash_save( - &Signature::now("Patchable", "noreply+patchable@stackable.tech").unwrap(), - "Existing work before checking out ", + tracing::info!( + patch.series = %series_file.display(), + "series file not found, treating as git mailbox" + ); + let mut patch_files = patch_dir + .read_dir() + .unwrap() + .map(|x| x.unwrap().path()) + .filter(|x| x.extension().is_some_and(|x| x == "patch")) + .collect::>(); + patch_files.sort(); + patch_files + }; + let mailsplit = raw_git_cmd(&product_version_repo) + .arg("mailsplit") + // mailsplit doesn't accept split arguments ("-o dir") + .arg(format!("-o{}", mailsplit_dir.path().to_str().unwrap())) + .arg("--") + .args(patch_files) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + if !mailsplit.status.success() { + panic!("failed to apply patches"); + } + let mailsplit_patch_count = str::from_utf8(&mailsplit.stdout) + .unwrap() + .trim() + .parse::() + .unwrap(); + let mut last_commit_id = product_version_repo + .revparse_single(&config.base) + .unwrap() + .id(); + for patch_i in 1..=mailsplit_patch_count { + // Matches the format emitted by git-mailsplit + let patch_mail_file_name = format!("{patch_i:04}"); + tracing::info!(patch.index = patch_mail_file_name, "parsing patch"); + let patch_split_msg_file = mailsplit_dir + .path() + .join(format!("{patch_mail_file_name}-msg")); + let patch_split_patch_file = mailsplit_dir + .path() + .join(format!("{patch_mail_file_name}-patch")); + let mailinfo = raw_git_cmd(&product_version_repo) + .arg("mailinfo") + .args([&patch_split_msg_file, &patch_split_patch_file]) + .stdin(File::open(mailsplit_dir.path().join(patch_mail_file_name)).unwrap()) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + if !mailinfo.status.success() { + panic!("failed to apply patches"); + } + let patch_info = str::from_utf8(&mailinfo.stdout).unwrap(); + + let mut author_name = None; + let mut author_email = None; + let mut date = None; + let mut subject = None; + for patch_info_line in patch_info.lines() { + if !patch_info_line.is_empty() { + match patch_info_line.split_once(": ").unwrap() { + ("Author", x) => author_name = Some(x), + ("Email", x) => author_email = Some(x), + ("Date", x) => date = Some(x), + ("Subject", x) => subject = Some(x), + (header, _) => panic!("unknown header type {header:?}"), + } + } + } + let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); + let author = Signature::new( + author_name.unwrap(), + author_email.unwrap(), + &git2::Time::new(date.unix_timestamp(), date.offset().whole_minutes().into()), + ) + .unwrap(); + let parent_commit = product_version_repo.find_commit(last_commit_id).unwrap(); + tracing::info!( + commit.base = %parent_commit.id(), + commit.subject = subject.unwrap(), + "applying patch" + ); + let patch_tree_id = product_version_repo + .apply_to_tree( + &parent_commit.tree().unwrap(), + &Diff::from_buffer(&std::fs::read(patch_split_patch_file).unwrap()) + .unwrap(), None, ) + .unwrap() + .write_tree_to(&product_version_repo) .unwrap(); - tracing::info!(%stash, "created stash"); - Some(stash) - }; + let msg = std::fs::read_to_string(patch_split_msg_file).unwrap(); + let full_msg = if msg.is_empty() { + subject.unwrap() + } else { + &format!("{}\n\n{msg}", subject.unwrap()) + }; + last_commit_id = product_version_repo + .commit( + None, + &author, + &author, + full_msg, + &product_version_repo.find_tree(patch_tree_id).unwrap(), + &[&parent_commit], + ) + .unwrap(); + tracing::info!( + commit.id = %last_commit_id, + "applied patch" + ); + } tracing::info!( worktree.branch = worktree_branch, + worktree.branch.commit = %last_commit_id, worktree.branch.base = config.base, - "checking out base commit into branch" + "checking out branch" ); // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime product_version_repo @@ -238,11 +363,7 @@ fn main() { let branch = product_version_repo .branch( &worktree_branch, - product_version_repo - .revparse_single(&config.base) - .unwrap() - .as_commit() - .unwrap(), + &product_version_repo.find_commit(last_commit_id).unwrap(), true, ) .unwrap() @@ -255,62 +376,6 @@ fn main() { .unwrap(); } - let patch_dir = ctx.patch_dir(); - tracing::info!( - patch.dir = %patch_dir.display(), - "applying patches" - ); - let series_file = patch_dir.join("series"); - let mut apply_cmd = raw_git_cmd(&product_version_repo); - if series_file.exists() { - tracing::info!( - patch.series = %series_file.display(), - "series file found, treating as stgit series" - ); - apply_cmd - .arg("am") - .arg(series_file) - .args(["--patch-format", "stgit-series"]); - } else { - tracing::info!( - patch.series = %series_file.display(), - "series file not found, treating as git mailbox" - ); - let mut patch_files = patch_dir - .read_dir() - .unwrap() - .map(|x| x.unwrap().path()) - .filter(|x| x.extension().is_some_and(|x| x == "patch")) - .collect::>(); - patch_files.sort(); - apply_cmd.arg("am").args(patch_files); - } - // Make commit hashes deterministic by removing variables that depend on the local environment - apply_cmd.args(["--committer-date-is-author-date"]); - if !apply_cmd.status().unwrap().success() { - panic!("failed to apply patches"); - } - - if let Some(stash) = stash { - let mut stash_index = None; - product_version_repo - .stash_foreach(|i, _, oid| { - if oid == &stash { - stash_index = Some(i); - true - } else { - false - } - }) - .unwrap(); - let stash_index = stash_index.unwrap(); - tracing::info!( - stash = %format_args!("stash@{{{stash_index}}}"), - "restoring stash" - ); - product_version_repo.stash_pop(stash_index, None).unwrap(); - } - tracing::info!( worktree.root = %product_worktree_root.display(), "worktree is ready!" From 710d6407dc230621f3bfe6510123610ea07953e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 6 Feb 2025 14:46:54 +0100 Subject: [PATCH 16/48] Mailsplit each patch file separately --- patchable/src/main.rs | 184 ++++++++++++++++++++++-------------------- 1 file changed, 96 insertions(+), 88 deletions(-) diff --git a/patchable/src/main.rs b/patchable/src/main.rs index f45012eba..4b16749d9 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -213,7 +213,6 @@ fn main() { // 2. Avoid touching the worktree until all patches have been applied // (letting us keep any dirty files in the worktree that don't conflict with the final switcheroo, // even if those files are modified by some of the patches) - let mailsplit_dir = tempdir().unwrap(); let series_file = patch_dir.join("series"); let patch_files = if series_file.exists() { tracing::info!( @@ -239,107 +238,116 @@ fn main() { patch_files.sort(); patch_files }; - let mailsplit = raw_git_cmd(&product_version_repo) - .arg("mailsplit") - // mailsplit doesn't accept split arguments ("-o dir") - .arg(format!("-o{}", mailsplit_dir.path().to_str().unwrap())) - .arg("--") - .args(patch_files) - .stderr(Stdio::inherit()) - .output() - .unwrap(); - if !mailsplit.status.success() { - panic!("failed to apply patches"); - } - let mailsplit_patch_count = str::from_utf8(&mailsplit.stdout) - .unwrap() - .trim() - .parse::() - .unwrap(); let mut last_commit_id = product_version_repo .revparse_single(&config.base) .unwrap() .id(); - for patch_i in 1..=mailsplit_patch_count { - // Matches the format emitted by git-mailsplit - let patch_mail_file_name = format!("{patch_i:04}"); - tracing::info!(patch.index = patch_mail_file_name, "parsing patch"); - let patch_split_msg_file = mailsplit_dir - .path() - .join(format!("{patch_mail_file_name}-msg")); - let patch_split_patch_file = mailsplit_dir - .path() - .join(format!("{patch_mail_file_name}-patch")); - let mailinfo = raw_git_cmd(&product_version_repo) - .arg("mailinfo") - .args([&patch_split_msg_file, &patch_split_patch_file]) - .stdin(File::open(mailsplit_dir.path().join(patch_mail_file_name)).unwrap()) + for patch_file in patch_files { + tracing::info!( + patch.file = %patch_file.display(), + "parsing patch" + ); + let mailsplit_dir = tempdir().unwrap(); + let mailsplit = raw_git_cmd(&product_version_repo) + .arg("mailsplit") + // mailsplit doesn't accept split arguments ("-o dir") + .arg(format!("-o{}", mailsplit_dir.path().to_str().unwrap())) + .arg("--") + .arg(patch_file) .stderr(Stdio::inherit()) .output() .unwrap(); - if !mailinfo.status.success() { + if !mailsplit.status.success() { panic!("failed to apply patches"); } - let patch_info = str::from_utf8(&mailinfo.stdout).unwrap(); + let mailsplit_patch_count = str::from_utf8(&mailsplit.stdout) + .unwrap() + .trim() + .parse::() + .unwrap(); + for patch_i in 1..=mailsplit_patch_count { + // Matches the format emitted by git-mailsplit + let patch_mail_file_name = format!("{patch_i:04}"); + let patch_split_msg_file = mailsplit_dir + .path() + .join(format!("{patch_mail_file_name}-msg")); + let patch_split_patch_file = mailsplit_dir + .path() + .join(format!("{patch_mail_file_name}-patch")); + let mailinfo = raw_git_cmd(&product_version_repo) + .arg("mailinfo") + .args([&patch_split_msg_file, &patch_split_patch_file]) + .stdin(File::open(mailsplit_dir.path().join(patch_mail_file_name)).unwrap()) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + if !mailinfo.status.success() { + panic!("failed to apply patches"); + } + let patch_info = str::from_utf8(&mailinfo.stdout).unwrap(); - let mut author_name = None; - let mut author_email = None; - let mut date = None; - let mut subject = None; - for patch_info_line in patch_info.lines() { - if !patch_info_line.is_empty() { - match patch_info_line.split_once(": ").unwrap() { - ("Author", x) => author_name = Some(x), - ("Email", x) => author_email = Some(x), - ("Date", x) => date = Some(x), - ("Subject", x) => subject = Some(x), - (header, _) => panic!("unknown header type {header:?}"), + let mut author_name = None; + let mut author_email = None; + let mut date = None; + let mut subject = None; + for patch_info_line in patch_info.lines() { + if !patch_info_line.is_empty() { + match patch_info_line.split_once(": ").unwrap() { + ("Author", x) => author_name = Some(x), + ("Email", x) => author_email = Some(x), + ("Date", x) => date = Some(x), + ("Subject", x) => subject = Some(x), + (header, _) => panic!("unknown header type {header:?}"), + } } } - } - let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); - let author = Signature::new( - author_name.unwrap(), - author_email.unwrap(), - &git2::Time::new(date.unix_timestamp(), date.offset().whole_minutes().into()), - ) - .unwrap(); - let parent_commit = product_version_repo.find_commit(last_commit_id).unwrap(); - tracing::info!( - commit.base = %parent_commit.id(), - commit.subject = subject.unwrap(), - "applying patch" - ); - let patch_tree_id = product_version_repo - .apply_to_tree( - &parent_commit.tree().unwrap(), - &Diff::from_buffer(&std::fs::read(patch_split_patch_file).unwrap()) - .unwrap(), - None, - ) - .unwrap() - .write_tree_to(&product_version_repo) - .unwrap(); - let msg = std::fs::read_to_string(patch_split_msg_file).unwrap(); - let full_msg = if msg.is_empty() { - subject.unwrap() - } else { - &format!("{}\n\n{msg}", subject.unwrap()) - }; - last_commit_id = product_version_repo - .commit( - None, - &author, - &author, - full_msg, - &product_version_repo.find_tree(patch_tree_id).unwrap(), - &[&parent_commit], + let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); + let author = Signature::new( + author_name.unwrap(), + author_email.unwrap(), + &git2::Time::new( + date.unix_timestamp(), + date.offset().whole_minutes().into(), + ), ) .unwrap(); - tracing::info!( - commit.id = %last_commit_id, - "applied patch" - ); + let parent_commit = product_version_repo.find_commit(last_commit_id).unwrap(); + tracing::info!( + commit.base = %parent_commit.id(), + commit.subject = subject.unwrap(), + "applying patch" + ); + let patch_tree_id = product_version_repo + .apply_to_tree( + &parent_commit.tree().unwrap(), + &Diff::from_buffer(&std::fs::read(patch_split_patch_file).unwrap()) + .unwrap(), + None, + ) + .unwrap() + .write_tree_to(&product_version_repo) + .unwrap(); + let msg = std::fs::read_to_string(patch_split_msg_file).unwrap(); + let full_msg = if msg.is_empty() { + subject.unwrap() + } else { + &format!("{}\n\n{msg}", subject.unwrap()) + }; + last_commit_id = product_version_repo + .commit( + None, + &author, + &author, + full_msg, + &product_version_repo.find_tree(patch_tree_id).unwrap(), + &[&parent_commit], + ) + .unwrap(); + tracing::info!( + commit.id = %last_commit_id, + "applied patch" + ); + } } tracing::info!( From 1228b7b7490570e9433a101ac7e7605c9809e88b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 6 Feb 2025 15:29:13 +0100 Subject: [PATCH 17/48] Simplify worktree checkout logic a bit --- patchable/src/main.rs | 154 ++++++++++++++++++------------------------ 1 file changed, 67 insertions(+), 87 deletions(-) diff --git a/patchable/src/main.rs b/patchable/src/main.rs index 4b16749d9..91ac29d0b 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -157,52 +157,6 @@ fn main() { } } - let product_worktree_root = ctx.worktree_root(); - let worktree_branch = ctx.worktree_branch(); - let product_version_repo = match Repository::open(&product_worktree_root) { - Ok(wt) => { - tracing::info!( - worktree.root = %product_worktree_root.display(), - "worktree found, resetting and reusing" - ); - wt - } - Err(err) => { - tracing::info!( - error = &err as &dyn std::error::Error, - worktree.root = %product_worktree_root.display(), - product.repository = %product_repo_root.display(), - "worktree not found, creating" - ); - std::fs::create_dir_all(product_worktree_root.parent().unwrap()).unwrap(); - let worktree_ref = product_repo - .branch( - &worktree_branch, - &product_repo - .revparse_single(&config.base) - .unwrap() - .peel_to_commit() - .unwrap(), - true, - ) - .unwrap() - .into_reference(); - Repository::open_from_worktree( - &product_repo - .worktree( - &ctx.pv.version, - &product_worktree_root, - Some(WorktreeAddOptions::new().reference(Some(&worktree_ref))), - ) - .unwrap(), - ) - .unwrap() - } - }; - - tracing::info!("cleaning up existing rebase state"); - product_version_repo.cleanup_state().unwrap(); - let patch_dir = ctx.patch_dir(); tracing::info!( patch.dir = %patch_dir.display(), @@ -238,17 +192,14 @@ fn main() { patch_files.sort(); patch_files }; - let mut last_commit_id = product_version_repo - .revparse_single(&config.base) - .unwrap() - .id(); + let mut last_commit_id = product_repo.revparse_single(&config.base).unwrap().id(); for patch_file in patch_files { tracing::info!( patch.file = %patch_file.display(), "parsing patch" ); let mailsplit_dir = tempdir().unwrap(); - let mailsplit = raw_git_cmd(&product_version_repo) + let mailsplit = raw_git_cmd(&product_repo) .arg("mailsplit") // mailsplit doesn't accept split arguments ("-o dir") .arg(format!("-o{}", mailsplit_dir.path().to_str().unwrap())) @@ -274,7 +225,7 @@ fn main() { let patch_split_patch_file = mailsplit_dir .path() .join(format!("{patch_mail_file_name}-patch")); - let mailinfo = raw_git_cmd(&product_version_repo) + let mailinfo = raw_git_cmd(&product_repo) .arg("mailinfo") .args([&patch_split_msg_file, &patch_split_patch_file]) .stdin(File::open(mailsplit_dir.path().join(patch_mail_file_name)).unwrap()) @@ -311,13 +262,13 @@ fn main() { ), ) .unwrap(); - let parent_commit = product_version_repo.find_commit(last_commit_id).unwrap(); + let parent_commit = product_repo.find_commit(last_commit_id).unwrap(); tracing::info!( commit.base = %parent_commit.id(), commit.subject = subject.unwrap(), "applying patch" ); - let patch_tree_id = product_version_repo + let patch_tree_id = product_repo .apply_to_tree( &parent_commit.tree().unwrap(), &Diff::from_buffer(&std::fs::read(patch_split_patch_file).unwrap()) @@ -325,7 +276,7 @@ fn main() { None, ) .unwrap() - .write_tree_to(&product_version_repo) + .write_tree_to(&product_repo) .unwrap(); let msg = std::fs::read_to_string(patch_split_msg_file).unwrap(); let full_msg = if msg.is_empty() { @@ -333,13 +284,13 @@ fn main() { } else { &format!("{}\n\n{msg}", subject.unwrap()) }; - last_commit_id = product_version_repo + last_commit_id = product_repo .commit( None, &author, &author, full_msg, - &product_version_repo.find_tree(patch_tree_id).unwrap(), + &product_repo.find_tree(patch_tree_id).unwrap(), &[&parent_commit], ) .unwrap(); @@ -350,38 +301,64 @@ fn main() { } } - tracing::info!( - worktree.branch = worktree_branch, - worktree.branch.commit = %last_commit_id, - worktree.branch.base = config.base, - "checking out branch" - ); - // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime - product_version_repo - .set_head_detached( - product_version_repo - .head() + let product_worktree_root = ctx.worktree_root(); + let worktree_branch = ctx.worktree_branch(); + match Repository::open(&product_worktree_root) { + Ok(worktree) => { + tracing::info!( + worktree.root = %product_worktree_root.display(), + worktree.branch = worktree_branch, + worktree.branch.commit = %last_commit_id, + worktree.branch.base = config.base, + "worktree found, reusing and resetting" + ); + // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime + worktree + .set_head_detached(worktree.head().unwrap().peel_to_commit().unwrap().id()) + .unwrap(); + let branch = worktree + .branch( + &worktree_branch, + &worktree.find_commit(last_commit_id).unwrap(), + true, + ) .unwrap() - .peel_to_commit() + .into_reference(); + worktree + .checkout_tree(&branch.peel(ObjectType::Commit).unwrap(), None) + .unwrap(); + worktree.set_head_bytes(branch.name_bytes()).unwrap(); + } + Err(err) => { + tracing::info!( + error = &err as &dyn std::error::Error, + worktree.root = %product_worktree_root.display(), + worktree.branch = worktree_branch, + worktree.branch.commit = %last_commit_id, + worktree.branch.base = config.base, + product.repository = %product_repo_root.display(), + "worktree not found, creating" + ); + std::fs::create_dir_all(product_worktree_root.parent().unwrap()).unwrap(); + let worktree_ref = product_repo + .branch( + &worktree_branch, + &product_repo.find_commit(last_commit_id).unwrap(), + true, + ) .unwrap() - .id(), - ) - .unwrap(); - { - let branch = product_version_repo - .branch( - &worktree_branch, - &product_version_repo.find_commit(last_commit_id).unwrap(), - true, + .into_reference(); + Repository::open_from_worktree( + &product_repo + .worktree( + &ctx.pv.version, + &product_worktree_root, + Some(WorktreeAddOptions::new().reference(Some(&worktree_ref))), + ) + .unwrap(), ) - .unwrap() - .into_reference(); - product_version_repo - .checkout_tree(&branch.peel(ObjectType::Commit).unwrap(), None) - .unwrap(); - product_version_repo - .set_head_bytes(branch.name_bytes()) .unwrap(); + } } tracing::info!( @@ -455,7 +432,10 @@ fn main() { /// Used for functionality that is not currently implemented by libgit2/gix. fn raw_git_cmd(repo: &Repository) -> std::process::Command { let mut cmd = std::process::Command::new("git"); - cmd.env("GIT_DIR", repo.path()) - .env("GIT_WORK_TREE", repo.workdir().unwrap()); + cmd.env("GIT_DIR", repo.path()); + cmd.env( + "GIT_WORK_TREE", + repo.workdir().unwrap_or(Path::new("/dev/null")), + ); cmd } From 2dbf6721f736ac96b0799ef91181960fa1341530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 6 Feb 2025 15:44:38 +0100 Subject: [PATCH 18/48] Forward logging from libgit2 --- Cargo.lock | 66 +++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- patchable/src/main.rs | 24 +++++++++++++++- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52b019db6..e33480770 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "0.6.18" @@ -445,6 +454,15 @@ version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "memchr" version = "2.7.4" @@ -553,6 +571,50 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + [[package]] name = "rustix" version = "0.38.44" @@ -797,10 +859,14 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] diff --git a/Cargo.toml b/Cargo.toml index 9f125fa62..a475ead32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,4 +10,4 @@ tempfile = "3.16.0" time = { version = "0.3.37", features = ["parsing"] } toml = "0.8.19" tracing = "0.1.41" -tracing-subscriber = "0.3.19" +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } diff --git a/patchable/src/main.rs b/patchable/src/main.rs index 91ac29d0b..6c7b1ee40 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -12,6 +12,7 @@ use git2::{ use serde::Deserialize; use tempfile::tempdir; use time::{format_description::well_known::Rfc2822, OffsetDateTime}; +use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _, EnvFilter}; #[derive(clap::Parser)] struct ProductVersion { @@ -90,7 +91,28 @@ enum Cmd { } fn main() { - tracing_subscriber::fmt().init(); + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer()) + .with( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing_subscriber::filter::LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .init(); + git2::trace_set(git2::TraceLevel::Trace, |level, msg| { + let msg = String::from_utf8_lossy(msg); + match level { + git2::TraceLevel::None | git2::TraceLevel::Fatal | git2::TraceLevel::Error => { + tracing::error!(target: "git", "{msg}") + } + git2::TraceLevel::Warn => tracing::warn!(target: "git", "{msg}"), + git2::TraceLevel::Info => tracing::info!(target: "git", "{msg}"), + git2::TraceLevel::Debug => tracing::debug!(target: "git", "{msg}"), + git2::TraceLevel::Trace => tracing::trace!(target: "git", "{msg}"), + } + }) + .unwrap(); + let opts = ::parse(); let images_repo = Repository::discover(".").unwrap(); let images_repo_root = images_repo.workdir().unwrap(); From 11d80a2d97e619303d5103ba5dbfbc17de25134c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 6 Feb 2025 20:58:03 +0100 Subject: [PATCH 19/48] Normalize commits before exporting --- ...0003-Stop-building-unused-extensions.patch | 2 +- ...ndencies-that-have-a-new-patch-relea.patch | 2 +- ...de-jackson-dataformat-xml-dependency.patch | 2 +- ...top-building-the-tar.gz-distribution.patch | 2 +- .../26.0.0/0007-Update-CycloneDX-plugin.patch | 2 +- .../26.0.0/0008-Fix-CVE-2024-36114.patch | 2 +- .../stackable/patches/26.0.0/0009-asdf.patch | 2 +- .../stackable/patches/26.0.0/0010-qwer.patch | 2 +- patchable/src/main.rs | 58 ++++++++++++++++++- 9 files changed, 63 insertions(+), 11 deletions(-) diff --git a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch index 8c93ad9a9..c59586fac 100644 --- a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch +++ b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch @@ -1,4 +1,4 @@ -From ca72868ea21177edd5e932d1915f7ad7f04de069 Mon Sep 17 00:00:00 2001 +From 85cacbcc47c88a56acd60d91fbf0412040523c8d Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Stop building unused extensions. diff --git a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch index 72c889c97..f3028c6d4 100644 --- a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch +++ b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch @@ -1,4 +1,4 @@ -From 74a9e778a1b06904566dd3c0ed7a4814459efdcb Mon Sep 17 00:00:00 2001 +From 4229d1c0d096e10dce72929224a7b4c2284fb417 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Updates all dependencies that have a new patch release available. diff --git a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch index 2fbbc586d..e65434626 100644 --- a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch +++ b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch @@ -1,4 +1,4 @@ -From 4276496b4df4c5e40df3b27bf9fc989b4ee9259b Mon Sep 17 00:00:00 2001 +From d55895a2525286a5198a3b327c3ce503bc852ead Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Include jackson-dataformat-xml dependency. diff --git a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch index acc3e500b..91f7c3edd 100644 --- a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch +++ b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch @@ -1,4 +1,4 @@ -From 92dcd86aec69c174a590d6854b428af93864c2c4 Mon Sep 17 00:00:00 2001 +From d1ae8732e2eee44abb5c831f5363c69e75e64a9a Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Stop building the tar.gz distribution. diff --git a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch index 5fd7179d0..eeed90fb8 100644 --- a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch +++ b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch @@ -1,4 +1,4 @@ -From f9520996498906692fd56bc7ffc35f385d14ce64 Mon Sep 17 00:00:00 2001 +From ff7d6a5ea07ea30653b47f6ef6844103a7ac3349 Mon Sep 17 00:00:00 2001 From: Lukas Voetmand Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Update CycloneDX plugin diff --git a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch index f7efbc9ea..c3628ffd3 100644 --- a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch +++ b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch @@ -1,4 +1,4 @@ -From 377483cdea7a88ddf17ea364b265e58e721b985a Mon Sep 17 00:00:00 2001 +From bdd52ae32874b686d6ddfa3179f6af787444662f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 12 Dec 2024 17:59:17 +0100 Subject: Fix CVE-2024-36114 diff --git a/druid/stackable/patches/26.0.0/0009-asdf.patch b/druid/stackable/patches/26.0.0/0009-asdf.patch index 04dedb8b5..4bb0252d0 100644 --- a/druid/stackable/patches/26.0.0/0009-asdf.patch +++ b/druid/stackable/patches/26.0.0/0009-asdf.patch @@ -1,4 +1,4 @@ -From 4d87008741b2b6356147b4b6ef648f49971f3cf9 Mon Sep 17 00:00:00 2001 +From 929707d95a423bda607cf8980c17bd7182c7b808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:26:17 +0100 Subject: asdf diff --git a/druid/stackable/patches/26.0.0/0010-qwer.patch b/druid/stackable/patches/26.0.0/0010-qwer.patch index 7ee4f5c49..95fbc9501 100644 --- a/druid/stackable/patches/26.0.0/0010-qwer.patch +++ b/druid/stackable/patches/26.0.0/0010-qwer.patch @@ -1,4 +1,4 @@ -From f793faeaa7182e7e997bd7571ac1e7e508b09077 Mon Sep 17 00:00:00 2001 +From 8de53941c8ad36144b598f14cd17a140333c499d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 12 Dec 2024 18:27:00 +0100 Subject: qwer diff --git a/patchable/src/main.rs b/patchable/src/main.rs index 6c7b1ee40..7dc0a3453 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -311,7 +311,7 @@ fn main() { None, &author, &author, - full_msg, + full_msg.trim(), &product_repo.find_tree(patch_tree_id).unwrap(), &[&parent_commit], ) @@ -402,6 +402,57 @@ fn main() { ); let product_version_repo = Repository::open(&product_worktree_root).unwrap(); + // Canonicalize commit messages and committer information, so that we generate the same commit IDs that we load in `patchable checkout`, + // even if we have rebased (etc) through them. + let head = product_version_repo + .head() + .unwrap() + .peel_to_commit() + .unwrap(); + let patch_base = product_version_repo + .revparse_single(&config.base) + .unwrap() + .peel_to_commit() + .unwrap(); + tracing::info!( + worktree.branch.base = %patch_base.id(), + worktree.branch.commit = %head.id(), + "canonicalizing commit history" + ); + let mut canonicalize_revwalk = product_version_repo.revwalk().unwrap(); + canonicalize_revwalk.push(head.id()).unwrap(); + canonicalize_revwalk.hide(patch_base.id()).unwrap(); + canonicalize_revwalk + .set_sorting(git2::Sort::TOPOLOGICAL | git2::Sort::REVERSE) + .unwrap(); + let mut last_canonical_commit = product_version_repo + .revparse_single(&config.base) + .unwrap() + .id(); + for base_commit in canonicalize_revwalk { + let base_commit = product_version_repo + .find_commit(base_commit.unwrap()) + .unwrap(); + let author = base_commit.author(); + last_canonical_commit = product_version_repo + .commit( + None, + &author, + &author, + base_commit.message().unwrap().trim(), + &base_commit.tree().unwrap(), + &[&product_version_repo + .find_commit(last_canonical_commit) + .unwrap()], + ) + .unwrap(); + } + tracing::info!( + worktree.branch.commit = %head.id(), + worktree.branch.commit.canonicalized = %last_canonical_commit, + "canonicalized commit" + ); + let patch_dir = ctx.patch_dir(); tracing::info!( patch.dir = %patch_dir.display(), @@ -423,12 +474,13 @@ fn main() { tracing::info!( patch.dir = %patch_dir.display(), worktree.root = %product_worktree_root.display(), - worktree.base = config.base, + worktree.branch.canonicalized = %last_canonical_commit, + worktree.branch.base = %patch_base.id(), "exporting commits since base" ); if !raw_git_cmd(&product_version_repo) .arg("format-patch") - .arg(&config.base) + .arg(format!("{}..{}", config.base, last_canonical_commit)) .arg("-o") .arg(&patch_dir) .arg("--base") From 12f5df5dc8eec2ff8749f83b33e2fdfda8b03a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 7 Feb 2025 10:09:25 +0100 Subject: [PATCH 20/48] Factor out implementation details from main() --- ...traces-of-the-druid-ranger-extension.patch | 2 - patchable/src/main.rs | 694 +++++++++--------- 2 files changed, 358 insertions(+), 338 deletions(-) diff --git a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch index 87f022916..3ab5b674f 100644 --- a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch +++ b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch @@ -42,8 +42,6 @@ index 0c6294f5ed..a33c6bd521 100644 extensions-core/druid-catalog extensions-core/testing-tools - -base-commit: 7cffb81a8e124d5f218f9af16ad685acf5e9c67c -- 2.48.1 diff --git a/patchable/src/main.rs b/patchable/src/main.rs index 7dc0a3453..e5049b994 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -6,13 +6,13 @@ use std::{ }; use git2::{ - Diff, FetchOptions, ObjectType, Repository, RepositoryInitOptions, Signature, + Diff, FetchOptions, ObjectType, Oid, Repository, RepositoryInitOptions, Signature, WorktreeAddOptions, }; use serde::Deserialize; use tempfile::tempdir; use time::{format_description::well_known::Rfc2822, OffsetDateTime}; -use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _, EnvFilter}; +use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; #[derive(clap::Parser)] struct ProductVersion { @@ -124,267 +124,27 @@ fn main() { }; let config = ctx.load_config(); let product_repo_root = ctx.repo(); - let product_repo = match Repository::open(&product_repo_root) { - Ok(repo) => { - tracing::info!( - product.repository = %product_repo_root.display(), - "product repository found, reusing" - ); - repo - } - Err(err) => { - tracing::info!( - error = &err as &dyn std::error::Error, - product.repository = %product_repo_root.display(), - "product repository not found, initializing" - ); - let repo = Repository::init_opts( - &product_repo_root, - RepositoryInitOptions::new() - .bare(true) - .initial_head("patchable-dummy") - .external_template(false), - ) - .unwrap(); - - repo - } - }; + let product_repo = tracing::info_span!( + "finding product repository", + product.repository = ?product_repo_root, + ) + .in_scope(|| ensure_bare_repo(&product_repo_root)); - match product_repo.revparse_single(&config.base) { - Ok(_) => tracing::info!( - worktree.branch.base = config.base, - "base commit exists, reusing" - ), - Err(err) => { - tracing::info!( - error = &err as &dyn std::error::Error, - worktree.branch.base = config.base, - product.upstream = config.upstream, - "base commit not found, fetching from upstream" - ); - product_repo - .remote_anonymous(&config.upstream) - .unwrap() - .fetch( - &[&config.base], - Some( - FetchOptions::new() - // TODO: could be 1, CLI option maybe? - .depth(0), - ), - None, - ) - .unwrap(); - } - } - - let patch_dir = ctx.patch_dir(); - tracing::info!( - patch.dir = %patch_dir.display(), - "applying patches" - ); - // This effectively reimplements git-am, but lets us: - // 1. Fake the committer information to match author (to ensure that we get deterministic committer IDs across machines) - // 2. Avoid touching the worktree until all patches have been applied - // (letting us keep any dirty files in the worktree that don't conflict with the final switcheroo, - // even if those files are modified by some of the patches) - let series_file = patch_dir.join("series"); - let patch_files = if series_file.exists() { - tracing::info!( - patch.series = %series_file.display(), - "series file found, treating as stgit series" - ); - std::fs::read_to_string(series_file) - .unwrap() - .lines() - .map(|file_name| patch_dir.join(file_name)) - .collect::>() - } else { - tracing::info!( - patch.series = %series_file.display(), - "series file not found, treating as git mailbox" - ); - let mut patch_files = patch_dir - .read_dir() - .unwrap() - .map(|x| x.unwrap().path()) - .filter(|x| x.extension().is_some_and(|x| x == "patch")) - .collect::>(); - patch_files.sort(); - patch_files - }; - let mut last_commit_id = product_repo.revparse_single(&config.base).unwrap().id(); - for patch_file in patch_files { - tracing::info!( - patch.file = %patch_file.display(), - "parsing patch" - ); - let mailsplit_dir = tempdir().unwrap(); - let mailsplit = raw_git_cmd(&product_repo) - .arg("mailsplit") - // mailsplit doesn't accept split arguments ("-o dir") - .arg(format!("-o{}", mailsplit_dir.path().to_str().unwrap())) - .arg("--") - .arg(patch_file) - .stderr(Stdio::inherit()) - .output() - .unwrap(); - if !mailsplit.status.success() { - panic!("failed to apply patches"); - } - let mailsplit_patch_count = str::from_utf8(&mailsplit.stdout) - .unwrap() - .trim() - .parse::() - .unwrap(); - for patch_i in 1..=mailsplit_patch_count { - // Matches the format emitted by git-mailsplit - let patch_mail_file_name = format!("{patch_i:04}"); - let patch_split_msg_file = mailsplit_dir - .path() - .join(format!("{patch_mail_file_name}-msg")); - let patch_split_patch_file = mailsplit_dir - .path() - .join(format!("{patch_mail_file_name}-patch")); - let mailinfo = raw_git_cmd(&product_repo) - .arg("mailinfo") - .args([&patch_split_msg_file, &patch_split_patch_file]) - .stdin(File::open(mailsplit_dir.path().join(patch_mail_file_name)).unwrap()) - .stderr(Stdio::inherit()) - .output() - .unwrap(); - if !mailinfo.status.success() { - panic!("failed to apply patches"); - } - let patch_info = str::from_utf8(&mailinfo.stdout).unwrap(); - - let mut author_name = None; - let mut author_email = None; - let mut date = None; - let mut subject = None; - for patch_info_line in patch_info.lines() { - if !patch_info_line.is_empty() { - match patch_info_line.split_once(": ").unwrap() { - ("Author", x) => author_name = Some(x), - ("Email", x) => author_email = Some(x), - ("Date", x) => date = Some(x), - ("Subject", x) => subject = Some(x), - (header, _) => panic!("unknown header type {header:?}"), - } - } - } - let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); - let author = Signature::new( - author_name.unwrap(), - author_email.unwrap(), - &git2::Time::new( - date.unix_timestamp(), - date.offset().whole_minutes().into(), - ), - ) - .unwrap(); - let parent_commit = product_repo.find_commit(last_commit_id).unwrap(); - tracing::info!( - commit.base = %parent_commit.id(), - commit.subject = subject.unwrap(), - "applying patch" - ); - let patch_tree_id = product_repo - .apply_to_tree( - &parent_commit.tree().unwrap(), - &Diff::from_buffer(&std::fs::read(patch_split_patch_file).unwrap()) - .unwrap(), - None, - ) - .unwrap() - .write_tree_to(&product_repo) - .unwrap(); - let msg = std::fs::read_to_string(patch_split_msg_file).unwrap(); - let full_msg = if msg.is_empty() { - subject.unwrap() - } else { - &format!("{}\n\n{msg}", subject.unwrap()) - }; - last_commit_id = product_repo - .commit( - None, - &author, - &author, - full_msg.trim(), - &product_repo.find_tree(patch_tree_id).unwrap(), - &[&parent_commit], - ) - .unwrap(); - tracing::info!( - commit.id = %last_commit_id, - "applied patch" - ); - } - } + let base_commit = + ensure_commit_exists_or_pull(&product_repo, &config.base, &config.upstream); + let patched_commit = apply_patches(&product_repo, &ctx.patch_dir(), base_commit); let product_worktree_root = ctx.worktree_root(); - let worktree_branch = ctx.worktree_branch(); - match Repository::open(&product_worktree_root) { - Ok(worktree) => { - tracing::info!( - worktree.root = %product_worktree_root.display(), - worktree.branch = worktree_branch, - worktree.branch.commit = %last_commit_id, - worktree.branch.base = config.base, - "worktree found, reusing and resetting" - ); - // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime - worktree - .set_head_detached(worktree.head().unwrap().peel_to_commit().unwrap().id()) - .unwrap(); - let branch = worktree - .branch( - &worktree_branch, - &worktree.find_commit(last_commit_id).unwrap(), - true, - ) - .unwrap() - .into_reference(); - worktree - .checkout_tree(&branch.peel(ObjectType::Commit).unwrap(), None) - .unwrap(); - worktree.set_head_bytes(branch.name_bytes()).unwrap(); - } - Err(err) => { - tracing::info!( - error = &err as &dyn std::error::Error, - worktree.root = %product_worktree_root.display(), - worktree.branch = worktree_branch, - worktree.branch.commit = %last_commit_id, - worktree.branch.base = config.base, - product.repository = %product_repo_root.display(), - "worktree not found, creating" - ); - std::fs::create_dir_all(product_worktree_root.parent().unwrap()).unwrap(); - let worktree_ref = product_repo - .branch( - &worktree_branch, - &product_repo.find_commit(last_commit_id).unwrap(), - true, - ) - .unwrap() - .into_reference(); - Repository::open_from_worktree( - &product_repo - .worktree( - &ctx.pv.version, - &product_worktree_root, - Some(WorktreeAddOptions::new().reference(Some(&worktree_ref))), - ) - .unwrap(), - ) - .unwrap(); - } - } + ensure_worktree_is_at( + &product_repo, + &ctx.pv.version, + &product_worktree_root, + &ctx.worktree_branch(), + patched_commit, + ); tracing::info!( - worktree.root = %product_worktree_root.display(), + worktree.root = ?product_worktree_root, "worktree is ready!" ); } @@ -397,108 +157,370 @@ fn main() { let product_worktree_root = ctx.worktree_root(); tracing::info!( - worktree.root = %product_worktree_root.display(), + worktree.root = ?product_worktree_root, "opening product worktree" ); let product_version_repo = Repository::open(&product_worktree_root).unwrap(); - // Canonicalize commit messages and committer information, so that we generate the same commit IDs that we load in `patchable checkout`, - // even if we have rebased (etc) through them. - let head = product_version_repo - .head() - .unwrap() - .peel_to_commit() - .unwrap(); - let patch_base = product_version_repo + let base_commit = product_version_repo .revparse_single(&config.base) .unwrap() .peel_to_commit() - .unwrap(); - tracing::info!( - worktree.branch.base = %patch_base.id(), - worktree.branch.commit = %head.id(), - "canonicalizing commit history" - ); - let mut canonicalize_revwalk = product_version_repo.revwalk().unwrap(); - canonicalize_revwalk.push(head.id()).unwrap(); - canonicalize_revwalk.hide(patch_base.id()).unwrap(); - canonicalize_revwalk - .set_sorting(git2::Sort::TOPOLOGICAL | git2::Sort::REVERSE) - .unwrap(); - let mut last_canonical_commit = product_version_repo - .revparse_single(&config.base) .unwrap() .id(); - for base_commit in canonicalize_revwalk { - let base_commit = product_version_repo - .find_commit(base_commit.unwrap()) - .unwrap(); - let author = base_commit.author(); - last_canonical_commit = product_version_repo - .commit( - None, - &author, - &author, - base_commit.message().unwrap().trim(), - &base_commit.tree().unwrap(), - &[&product_version_repo - .find_commit(last_canonical_commit) - .unwrap()], - ) - .unwrap(); - } - tracing::info!( - worktree.branch.commit = %head.id(), - worktree.branch.commit.canonicalized = %last_canonical_commit, - "canonicalized commit" + let canonical_leaf_commit = canonicalize_commit_history( + &product_version_repo, + base_commit, + product_version_repo + .head() + .unwrap() + .peel_to_commit() + .unwrap() + .id(), ); let patch_dir = ctx.patch_dir(); + format_patches( + &product_version_repo, + &patch_dir, + base_commit, + canonical_leaf_commit, + ); + tracing::info!( - patch.dir = %patch_dir.display(), - "deleting existing patch files" + patch.dir = ?patch_dir, + "worktree is exported!" ); - // git format-patch is happy to overwrite existing files, - // but we also want to delete removed (or renamed) patch files. - for entry in patch_dir.read_dir().unwrap() { - let path = entry.unwrap().path(); - // git format-patch emits the mailbox format, not stgit(/quilt), - // so also remove markers that make it look like that - if path.file_name().is_some_and(|x| x == "series") - || path.extension().is_some_and(|x| x == "patch") - { - std::fs::remove_file(path).unwrap(); - } - } + } + } +} + +/// Open the Git repository at `path`, creating it if it doesn't already exist. +#[tracing::instrument] +fn ensure_bare_repo(path: &Path) -> Repository { + match Repository::open(path) { + Ok(repo) => { + tracing::info!("repository found, reusing"); + repo + } + Err(err) => { + tracing::info!( + error = &err as &dyn std::error::Error, + "repository not found, initializing" + ); + let repo = Repository::init_opts( + path, + RepositoryInitOptions::new() + .bare(true) + .external_template(false), + ) + .unwrap(); + + repo + } + } +} + +/// Pull `commit` from `upstream_url`, if it doesn't already exist. +#[tracing::instrument(skip(repo))] +fn ensure_commit_exists_or_pull(repo: &Repository, commit: &str, upstream_url: &str) -> Oid { + match repo.revparse_single(commit) { + Ok(commit_obj) => { + tracing::info!("base commit exists, reusing"); + commit_obj + } + Err(err) => { + tracing::info!( + error = &err as &dyn std::error::Error, + "base commit not found, fetching from upstream" + ); + repo.remote_anonymous(upstream_url) + .unwrap() + .fetch( + &[commit], + Some( + FetchOptions::new() + // TODO: could be 1, CLI option maybe? + .depth(0), + ), + Some("fetching patchable base commit"), + ) + .unwrap(); + tracing::info!("fetched base commit"); + repo.revparse_single(commit).unwrap() + } + } + .peel_to_commit() + .unwrap() + .id() +} +/// Ensure that the worktree at `worktree_root` exists and is checked out at `branch`. +/// +/// The worktree will be created if necessary, and the branch will be created or reset to `commit`. +#[tracing::instrument(skip(repo))] +fn ensure_worktree_is_at( + repo: &Repository, + worktree_name: &str, + worktree_root: &Path, + branch: &str, + commit: Oid, +) { + match Repository::open(worktree_root) { + Ok(worktree) => { + tracing::info!("worktree found, reusing and resetting"); + // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime + worktree + .set_head_detached(worktree.head().unwrap().peel_to_commit().unwrap().id()) + .unwrap(); + let branch = worktree + .branch(branch, &worktree.find_commit(commit).unwrap(), true) + .unwrap() + .into_reference(); + worktree + .checkout_tree(&branch.peel(ObjectType::Commit).unwrap(), None) + .unwrap(); + worktree.set_head_bytes(branch.name_bytes()).unwrap(); + } + Err(err) => { tracing::info!( - patch.dir = %patch_dir.display(), - worktree.root = %product_worktree_root.display(), - worktree.branch.canonicalized = %last_canonical_commit, - worktree.branch.base = %patch_base.id(), - "exporting commits since base" + error = &err as &dyn std::error::Error, + "worktree not found, creating" ); - if !raw_git_cmd(&product_version_repo) - .arg("format-patch") - .arg(format!("{}..{}", config.base, last_canonical_commit)) - .arg("-o") - .arg(&patch_dir) - .arg("--base") - .arg(&config.base) - .arg("--keep-subject") - .status() + std::fs::create_dir_all(worktree_root.parent().unwrap()).unwrap(); + let worktree_ref = repo + .branch(branch, &repo.find_commit(commit).unwrap(), true) .unwrap() - .success() - { - panic!("failed to format patches"); + .into_reference(); + repo.worktree( + worktree_name, + worktree_root, + Some(WorktreeAddOptions::new().reference(Some(&worktree_ref))), + ) + .unwrap(); + } + } +} + +/// Apply the patches in `patch_dir` to `base_commit`. +/// +/// Does not modify the worktree or branch(es). Use [`ensure_worktree_is_at`] to check out the commit afterwards. +/// +/// This effectively reimplements git-am, but lets us: +/// 1. Fake the committer information to match author (to ensure that we get deterministic committer IDs across machines) +/// 2. Avoid touching the worktree until all patches have been applied +/// (letting us keep any dirty files in the worktree that don't conflict with the final switcheroo, +/// even if those files are modified by some of the patches) +#[tracing::instrument(skip(repo))] +fn apply_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid) -> Oid { + tracing::info!( + patch.dir = %patch_dir.display(), + "applying patches" + ); + let series_file = patch_dir.join("series"); + let patch_files = if series_file.exists() { + tracing::info!( + patch.series = %series_file.display(), + "series file found, treating as stgit series" + ); + std::fs::read_to_string(series_file) + .unwrap() + .lines() + .map(|file_name| patch_dir.join(file_name)) + .collect::>() + } else { + tracing::info!( + patch.series = %series_file.display(), + "series file not found, treating as git mailbox" + ); + let mut patch_files = patch_dir + .read_dir() + .unwrap() + .map(|x| x.unwrap().path()) + .filter(|x| x.extension().is_some_and(|x| x == "patch")) + .collect::>(); + patch_files.sort(); + patch_files + }; + let mut last_commit_id = base_commit; + for patch_file in patch_files { + tracing::info!( + patch.file = %patch_file.display(), + "parsing patch" + ); + let mailsplit_dir = tempdir().unwrap(); + let mailsplit = raw_git_cmd(repo) + .arg("mailsplit") + // mailsplit doesn't accept split arguments ("-o dir") + .arg(format!("-o{}", mailsplit_dir.path().to_str().unwrap())) + .arg("--") + .arg(patch_file) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + if !mailsplit.status.success() { + panic!("failed to apply patches"); + } + let mailsplit_patch_count = str::from_utf8(&mailsplit.stdout) + .unwrap() + .trim() + .parse::() + .unwrap(); + for patch_i in 1..=mailsplit_patch_count { + // Matches the format emitted by git-mailsplit + let patch_mail_file_name = format!("{patch_i:04}"); + let patch_split_msg_file = mailsplit_dir + .path() + .join(format!("{patch_mail_file_name}-msg")); + let patch_split_patch_file = mailsplit_dir + .path() + .join(format!("{patch_mail_file_name}-patch")); + let mailinfo = raw_git_cmd(repo) + .arg("mailinfo") + .args([&patch_split_msg_file, &patch_split_patch_file]) + .stdin(File::open(mailsplit_dir.path().join(patch_mail_file_name)).unwrap()) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + if !mailinfo.status.success() { + panic!("failed to apply patches"); } + let patch_info = str::from_utf8(&mailinfo.stdout).unwrap(); + let mut author_name = None; + let mut author_email = None; + let mut date = None; + let mut subject = None; + for patch_info_line in patch_info.lines() { + if !patch_info_line.is_empty() { + match patch_info_line.split_once(": ").unwrap() { + ("Author", x) => author_name = Some(x), + ("Email", x) => author_email = Some(x), + ("Date", x) => date = Some(x), + ("Subject", x) => subject = Some(x), + (header, _) => panic!("unknown header type {header:?}"), + } + } + } + let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); + let author = Signature::new( + author_name.unwrap(), + author_email.unwrap(), + &git2::Time::new(date.unix_timestamp(), date.offset().whole_minutes().into()), + ) + .unwrap(); + let parent_commit = repo.find_commit(last_commit_id).unwrap(); tracing::info!( - patch.dir = %patch_dir.display(), - "worktree is exported!" + commit.base = %parent_commit.id(), + commit.subject = subject.unwrap(), + "applying patch" ); + let patch_tree_id = repo + .apply_to_tree( + &parent_commit.tree().unwrap(), + &Diff::from_buffer(&std::fs::read(patch_split_patch_file).unwrap()).unwrap(), + None, + ) + .unwrap() + .write_tree_to(repo) + .unwrap(); + let msg = std::fs::read_to_string(patch_split_msg_file).unwrap(); + let full_msg = if msg.is_empty() { + subject.unwrap() + } else { + &format!("{}\n\n{msg}", subject.unwrap()) + }; + last_commit_id = repo + .commit( + None, + &author, + &author, + full_msg.trim(), + &repo.find_tree(patch_tree_id).unwrap(), + &[&parent_commit], + ) + .unwrap(); + tracing::info!( + commit.id = %last_commit_id, + "applied patch" + ); + } + } + last_commit_id +} + +/// Canonicalize commits for all commits between `base_commit` (exclusive) and `leaf_commit` (inclusive). +/// +/// Does not modify the worktree. +/// +/// This should generate the same commit IDs we would have loaded in [`apply_patches`]. +/// +/// This is required to avoid commit ID churn because of author/committer mismatch +/// (generated whenever a commit is modified after the initial commit), and +/// whitespace mangling in git-format-patch. +#[tracing::instrument(skip(repo))] +fn canonicalize_commit_history(repo: &Repository, base_commit: Oid, leaf_commit: Oid) -> Oid { + tracing::info!("canonicalizing commit history"); + let mut canonicalize_revwalk = repo.revwalk().unwrap(); + canonicalize_revwalk.push(leaf_commit).unwrap(); + canonicalize_revwalk.hide(base_commit).unwrap(); + canonicalize_revwalk + .set_sorting(git2::Sort::TOPOLOGICAL | git2::Sort::REVERSE) + .unwrap(); + let mut last_canonical_commit = base_commit; + for original in canonicalize_revwalk { + let original = repo.find_commit(original.unwrap()).unwrap(); + let author = original.author(); + last_canonical_commit = repo + .commit( + None, + &author, + &author, + original.message().unwrap().trim(), + &original.tree().unwrap(), + &[&repo.find_commit(last_canonical_commit).unwrap()], + ) + .unwrap(); + } + tracing::info!( + leaf_commit.canonical = %last_canonical_commit, + "canonicalized commit history" + ); + last_canonical_commit +} + +/// Formats the commits between `base_commit` (exclusive) and `leaf_commit` (inclusive) as patches in `patch_dir`. +/// +/// Deletes any existing patch files in `patch_dir`. +#[tracing::instrument(skip(repo))] +fn format_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid, leaf_commit: Oid) { + tracing::info!("deleting existing patch files"); + // git format-patch is happy to overwrite existing files, + // but we also want to delete removed (or renamed) patch files. + for entry in patch_dir.read_dir().unwrap() { + let path = entry.unwrap().path(); + // git format-patch emits the mailbox format, not stgit(/quilt), + // so also remove markers that make it look like that + if path.file_name().is_some_and(|x| x == "series") + || path.extension().is_some_and(|x| x == "patch") + { + std::fs::remove_file(path).unwrap(); } } + + tracing::info!("exporting commits since base"); + if !raw_git_cmd(repo) + .arg("format-patch") + .arg(format!("{base_commit}..{leaf_commit}")) + .arg("-o") + .arg(patch_dir) + .arg("--keep-subject") + .status() + .unwrap() + .success() + { + panic!("failed to format patches"); + } } /// Runs a raw git command in the environment of a Git repository. From cdd4427fb6ee719228f2477e22e2f32b6391850b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 7 Feb 2025 10:17:31 +0100 Subject: [PATCH 21/48] Modularize --- patchable/src/main.rs | 370 ++--------------------------------------- patchable/src/patch.rs | 228 +++++++++++++++++++++++++ patchable/src/repo.rs | 110 ++++++++++++ patchable/src/utils.rs | 16 ++ 4 files changed, 366 insertions(+), 358 deletions(-) create mode 100644 patchable/src/patch.rs create mode 100644 patchable/src/repo.rs create mode 100644 patchable/src/utils.rs diff --git a/patchable/src/main.rs b/patchable/src/main.rs index e5049b994..bf6364d3e 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -1,17 +1,12 @@ +mod patch; +mod repo; +mod utils; + use core::str; -use std::{ - fs::File, - path::{Path, PathBuf}, - process::Stdio, -}; +use std::path::{Path, PathBuf}; -use git2::{ - Diff, FetchOptions, ObjectType, Oid, Repository, RepositoryInitOptions, Signature, - WorktreeAddOptions, -}; +use git2::Repository; use serde::Deserialize; -use tempfile::tempdir; -use time::{format_description::well_known::Rfc2822, OffsetDateTime}; use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; #[derive(clap::Parser)] @@ -128,14 +123,14 @@ fn main() { "finding product repository", product.repository = ?product_repo_root, ) - .in_scope(|| ensure_bare_repo(&product_repo_root)); + .in_scope(|| repo::ensure_bare_repo(&product_repo_root)); let base_commit = - ensure_commit_exists_or_pull(&product_repo, &config.base, &config.upstream); - let patched_commit = apply_patches(&product_repo, &ctx.patch_dir(), base_commit); + repo::ensure_commit_exists_or_pull(&product_repo, &config.base, &config.upstream); + let patched_commit = patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit); let product_worktree_root = ctx.worktree_root(); - ensure_worktree_is_at( + repo::ensure_worktree_is_at( &product_repo, &ctx.pv.version, &product_worktree_root, @@ -168,7 +163,7 @@ fn main() { .peel_to_commit() .unwrap() .id(); - let canonical_leaf_commit = canonicalize_commit_history( + let canonical_leaf_commit = patch::canonicalize_commit_history( &product_version_repo, base_commit, product_version_repo @@ -180,7 +175,7 @@ fn main() { ); let patch_dir = ctx.patch_dir(); - format_patches( + patch::format_patches( &product_version_repo, &patch_dir, base_commit, @@ -194,344 +189,3 @@ fn main() { } } } - -/// Open the Git repository at `path`, creating it if it doesn't already exist. -#[tracing::instrument] -fn ensure_bare_repo(path: &Path) -> Repository { - match Repository::open(path) { - Ok(repo) => { - tracing::info!("repository found, reusing"); - repo - } - Err(err) => { - tracing::info!( - error = &err as &dyn std::error::Error, - "repository not found, initializing" - ); - let repo = Repository::init_opts( - path, - RepositoryInitOptions::new() - .bare(true) - .external_template(false), - ) - .unwrap(); - - repo - } - } -} - -/// Pull `commit` from `upstream_url`, if it doesn't already exist. -#[tracing::instrument(skip(repo))] -fn ensure_commit_exists_or_pull(repo: &Repository, commit: &str, upstream_url: &str) -> Oid { - match repo.revparse_single(commit) { - Ok(commit_obj) => { - tracing::info!("base commit exists, reusing"); - commit_obj - } - Err(err) => { - tracing::info!( - error = &err as &dyn std::error::Error, - "base commit not found, fetching from upstream" - ); - repo.remote_anonymous(upstream_url) - .unwrap() - .fetch( - &[commit], - Some( - FetchOptions::new() - // TODO: could be 1, CLI option maybe? - .depth(0), - ), - Some("fetching patchable base commit"), - ) - .unwrap(); - tracing::info!("fetched base commit"); - repo.revparse_single(commit).unwrap() - } - } - .peel_to_commit() - .unwrap() - .id() -} - -/// Ensure that the worktree at `worktree_root` exists and is checked out at `branch`. -/// -/// The worktree will be created if necessary, and the branch will be created or reset to `commit`. -#[tracing::instrument(skip(repo))] -fn ensure_worktree_is_at( - repo: &Repository, - worktree_name: &str, - worktree_root: &Path, - branch: &str, - commit: Oid, -) { - match Repository::open(worktree_root) { - Ok(worktree) => { - tracing::info!("worktree found, reusing and resetting"); - // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime - worktree - .set_head_detached(worktree.head().unwrap().peel_to_commit().unwrap().id()) - .unwrap(); - let branch = worktree - .branch(branch, &worktree.find_commit(commit).unwrap(), true) - .unwrap() - .into_reference(); - worktree - .checkout_tree(&branch.peel(ObjectType::Commit).unwrap(), None) - .unwrap(); - worktree.set_head_bytes(branch.name_bytes()).unwrap(); - } - Err(err) => { - tracing::info!( - error = &err as &dyn std::error::Error, - "worktree not found, creating" - ); - std::fs::create_dir_all(worktree_root.parent().unwrap()).unwrap(); - let worktree_ref = repo - .branch(branch, &repo.find_commit(commit).unwrap(), true) - .unwrap() - .into_reference(); - repo.worktree( - worktree_name, - worktree_root, - Some(WorktreeAddOptions::new().reference(Some(&worktree_ref))), - ) - .unwrap(); - } - } -} - -/// Apply the patches in `patch_dir` to `base_commit`. -/// -/// Does not modify the worktree or branch(es). Use [`ensure_worktree_is_at`] to check out the commit afterwards. -/// -/// This effectively reimplements git-am, but lets us: -/// 1. Fake the committer information to match author (to ensure that we get deterministic committer IDs across machines) -/// 2. Avoid touching the worktree until all patches have been applied -/// (letting us keep any dirty files in the worktree that don't conflict with the final switcheroo, -/// even if those files are modified by some of the patches) -#[tracing::instrument(skip(repo))] -fn apply_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid) -> Oid { - tracing::info!( - patch.dir = %patch_dir.display(), - "applying patches" - ); - let series_file = patch_dir.join("series"); - let patch_files = if series_file.exists() { - tracing::info!( - patch.series = %series_file.display(), - "series file found, treating as stgit series" - ); - std::fs::read_to_string(series_file) - .unwrap() - .lines() - .map(|file_name| patch_dir.join(file_name)) - .collect::>() - } else { - tracing::info!( - patch.series = %series_file.display(), - "series file not found, treating as git mailbox" - ); - let mut patch_files = patch_dir - .read_dir() - .unwrap() - .map(|x| x.unwrap().path()) - .filter(|x| x.extension().is_some_and(|x| x == "patch")) - .collect::>(); - patch_files.sort(); - patch_files - }; - let mut last_commit_id = base_commit; - for patch_file in patch_files { - tracing::info!( - patch.file = %patch_file.display(), - "parsing patch" - ); - let mailsplit_dir = tempdir().unwrap(); - let mailsplit = raw_git_cmd(repo) - .arg("mailsplit") - // mailsplit doesn't accept split arguments ("-o dir") - .arg(format!("-o{}", mailsplit_dir.path().to_str().unwrap())) - .arg("--") - .arg(patch_file) - .stderr(Stdio::inherit()) - .output() - .unwrap(); - if !mailsplit.status.success() { - panic!("failed to apply patches"); - } - let mailsplit_patch_count = str::from_utf8(&mailsplit.stdout) - .unwrap() - .trim() - .parse::() - .unwrap(); - for patch_i in 1..=mailsplit_patch_count { - // Matches the format emitted by git-mailsplit - let patch_mail_file_name = format!("{patch_i:04}"); - let patch_split_msg_file = mailsplit_dir - .path() - .join(format!("{patch_mail_file_name}-msg")); - let patch_split_patch_file = mailsplit_dir - .path() - .join(format!("{patch_mail_file_name}-patch")); - let mailinfo = raw_git_cmd(repo) - .arg("mailinfo") - .args([&patch_split_msg_file, &patch_split_patch_file]) - .stdin(File::open(mailsplit_dir.path().join(patch_mail_file_name)).unwrap()) - .stderr(Stdio::inherit()) - .output() - .unwrap(); - if !mailinfo.status.success() { - panic!("failed to apply patches"); - } - let patch_info = str::from_utf8(&mailinfo.stdout).unwrap(); - - let mut author_name = None; - let mut author_email = None; - let mut date = None; - let mut subject = None; - for patch_info_line in patch_info.lines() { - if !patch_info_line.is_empty() { - match patch_info_line.split_once(": ").unwrap() { - ("Author", x) => author_name = Some(x), - ("Email", x) => author_email = Some(x), - ("Date", x) => date = Some(x), - ("Subject", x) => subject = Some(x), - (header, _) => panic!("unknown header type {header:?}"), - } - } - } - let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); - let author = Signature::new( - author_name.unwrap(), - author_email.unwrap(), - &git2::Time::new(date.unix_timestamp(), date.offset().whole_minutes().into()), - ) - .unwrap(); - let parent_commit = repo.find_commit(last_commit_id).unwrap(); - tracing::info!( - commit.base = %parent_commit.id(), - commit.subject = subject.unwrap(), - "applying patch" - ); - let patch_tree_id = repo - .apply_to_tree( - &parent_commit.tree().unwrap(), - &Diff::from_buffer(&std::fs::read(patch_split_patch_file).unwrap()).unwrap(), - None, - ) - .unwrap() - .write_tree_to(repo) - .unwrap(); - let msg = std::fs::read_to_string(patch_split_msg_file).unwrap(); - let full_msg = if msg.is_empty() { - subject.unwrap() - } else { - &format!("{}\n\n{msg}", subject.unwrap()) - }; - last_commit_id = repo - .commit( - None, - &author, - &author, - full_msg.trim(), - &repo.find_tree(patch_tree_id).unwrap(), - &[&parent_commit], - ) - .unwrap(); - tracing::info!( - commit.id = %last_commit_id, - "applied patch" - ); - } - } - last_commit_id -} - -/// Canonicalize commits for all commits between `base_commit` (exclusive) and `leaf_commit` (inclusive). -/// -/// Does not modify the worktree. -/// -/// This should generate the same commit IDs we would have loaded in [`apply_patches`]. -/// -/// This is required to avoid commit ID churn because of author/committer mismatch -/// (generated whenever a commit is modified after the initial commit), and -/// whitespace mangling in git-format-patch. -#[tracing::instrument(skip(repo))] -fn canonicalize_commit_history(repo: &Repository, base_commit: Oid, leaf_commit: Oid) -> Oid { - tracing::info!("canonicalizing commit history"); - let mut canonicalize_revwalk = repo.revwalk().unwrap(); - canonicalize_revwalk.push(leaf_commit).unwrap(); - canonicalize_revwalk.hide(base_commit).unwrap(); - canonicalize_revwalk - .set_sorting(git2::Sort::TOPOLOGICAL | git2::Sort::REVERSE) - .unwrap(); - let mut last_canonical_commit = base_commit; - for original in canonicalize_revwalk { - let original = repo.find_commit(original.unwrap()).unwrap(); - let author = original.author(); - last_canonical_commit = repo - .commit( - None, - &author, - &author, - original.message().unwrap().trim(), - &original.tree().unwrap(), - &[&repo.find_commit(last_canonical_commit).unwrap()], - ) - .unwrap(); - } - tracing::info!( - leaf_commit.canonical = %last_canonical_commit, - "canonicalized commit history" - ); - last_canonical_commit -} - -/// Formats the commits between `base_commit` (exclusive) and `leaf_commit` (inclusive) as patches in `patch_dir`. -/// -/// Deletes any existing patch files in `patch_dir`. -#[tracing::instrument(skip(repo))] -fn format_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid, leaf_commit: Oid) { - tracing::info!("deleting existing patch files"); - // git format-patch is happy to overwrite existing files, - // but we also want to delete removed (or renamed) patch files. - for entry in patch_dir.read_dir().unwrap() { - let path = entry.unwrap().path(); - // git format-patch emits the mailbox format, not stgit(/quilt), - // so also remove markers that make it look like that - if path.file_name().is_some_and(|x| x == "series") - || path.extension().is_some_and(|x| x == "patch") - { - std::fs::remove_file(path).unwrap(); - } - } - - tracing::info!("exporting commits since base"); - if !raw_git_cmd(repo) - .arg("format-patch") - .arg(format!("{base_commit}..{leaf_commit}")) - .arg("-o") - .arg(patch_dir) - .arg("--keep-subject") - .status() - .unwrap() - .success() - { - panic!("failed to format patches"); - } -} - -/// Runs a raw git command in the environment of a Git repository. -/// -/// Used for functionality that is not currently implemented by libgit2/gix. -fn raw_git_cmd(repo: &Repository) -> std::process::Command { - let mut cmd = std::process::Command::new("git"); - cmd.env("GIT_DIR", repo.path()); - cmd.env( - "GIT_WORK_TREE", - repo.workdir().unwrap_or(Path::new("/dev/null")), - ); - cmd -} diff --git a/patchable/src/patch.rs b/patchable/src/patch.rs new file mode 100644 index 000000000..f44b42f55 --- /dev/null +++ b/patchable/src/patch.rs @@ -0,0 +1,228 @@ +use std::{fs::File, path::Path, process::Stdio}; + +use git2::{Diff, Oid, Repository, Signature}; +use tempfile::tempdir; +use time::{format_description::well_known::Rfc2822, OffsetDateTime}; + +use crate::utils::raw_git_cmd; + +/// Apply the patches in `patch_dir` to `base_commit`. +/// +/// Does not modify the worktree or branch(es). Use [`ensure_worktree_is_at`] to check out the commit afterwards. +/// +/// This effectively reimplements git-am, but lets us: +/// 1. Fake the committer information to match author (to ensure that we get deterministic committer IDs across machines) +/// 2. Avoid touching the worktree until all patches have been applied +/// (letting us keep any dirty files in the worktree that don't conflict with the final switcheroo, +/// even if those files are modified by some of the patches) +#[tracing::instrument(skip(repo))] +pub fn apply_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid) -> Oid { + tracing::info!( + patch.dir = %patch_dir.display(), + "applying patches" + ); + let series_file = patch_dir.join("series"); + let patch_files = if series_file.exists() { + tracing::info!( + patch.series = %series_file.display(), + "series file found, treating as stgit series" + ); + std::fs::read_to_string(series_file) + .unwrap() + .lines() + .map(|file_name| patch_dir.join(file_name)) + .collect::>() + } else { + tracing::info!( + patch.series = %series_file.display(), + "series file not found, treating as git mailbox" + ); + let mut patch_files = patch_dir + .read_dir() + .unwrap() + .map(|x| x.unwrap().path()) + .filter(|x| x.extension().is_some_and(|x| x == "patch")) + .collect::>(); + patch_files.sort(); + patch_files + }; + let mut last_commit_id = base_commit; + for patch_file in patch_files { + tracing::info!( + patch.file = %patch_file.display(), + "parsing patch" + ); + let mailsplit_dir = tempdir().unwrap(); + let mailsplit = raw_git_cmd(repo) + .arg("mailsplit") + // mailsplit doesn't accept split arguments ("-o dir") + .arg(format!("-o{}", mailsplit_dir.path().to_str().unwrap())) + .arg("--") + .arg(patch_file) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + if !mailsplit.status.success() { + panic!("failed to apply patches"); + } + let mailsplit_patch_count = std::str::from_utf8(&mailsplit.stdout) + .unwrap() + .trim() + .parse::() + .unwrap(); + for patch_i in 1..=mailsplit_patch_count { + // Matches the format emitted by git-mailsplit + let patch_mail_file_name = format!("{patch_i:04}"); + let patch_split_msg_file = mailsplit_dir + .path() + .join(format!("{patch_mail_file_name}-msg")); + let patch_split_patch_file = mailsplit_dir + .path() + .join(format!("{patch_mail_file_name}-patch")); + let mailinfo = raw_git_cmd(repo) + .arg("mailinfo") + .args([&patch_split_msg_file, &patch_split_patch_file]) + .stdin(File::open(mailsplit_dir.path().join(patch_mail_file_name)).unwrap()) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + if !mailinfo.status.success() { + panic!("failed to apply patches"); + } + let patch_info = std::str::from_utf8(&mailinfo.stdout).unwrap(); + + let mut author_name = None; + let mut author_email = None; + let mut date = None; + let mut subject = None; + for patch_info_line in patch_info.lines() { + if !patch_info_line.is_empty() { + match patch_info_line.split_once(": ").unwrap() { + ("Author", x) => author_name = Some(x), + ("Email", x) => author_email = Some(x), + ("Date", x) => date = Some(x), + ("Subject", x) => subject = Some(x), + (header, _) => panic!("unknown header type {header:?}"), + } + } + } + let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); + let author = Signature::new( + author_name.unwrap(), + author_email.unwrap(), + &git2::Time::new(date.unix_timestamp(), date.offset().whole_minutes().into()), + ) + .unwrap(); + let parent_commit = repo.find_commit(last_commit_id).unwrap(); + tracing::info!( + commit.base = %parent_commit.id(), + commit.subject = subject.unwrap(), + "applying patch" + ); + let patch_tree_id = repo + .apply_to_tree( + &parent_commit.tree().unwrap(), + &Diff::from_buffer(&std::fs::read(patch_split_patch_file).unwrap()).unwrap(), + None, + ) + .unwrap() + .write_tree_to(repo) + .unwrap(); + let msg = std::fs::read_to_string(patch_split_msg_file).unwrap(); + let full_msg = if msg.is_empty() { + subject.unwrap() + } else { + &format!("{}\n\n{msg}", subject.unwrap()) + }; + last_commit_id = repo + .commit( + None, + &author, + &author, + full_msg.trim(), + &repo.find_tree(patch_tree_id).unwrap(), + &[&parent_commit], + ) + .unwrap(); + tracing::info!( + commit.id = %last_commit_id, + "applied patch" + ); + } + } + last_commit_id +} + +/// Canonicalize commits for all commits between `base_commit` (exclusive) and `leaf_commit` (inclusive). +/// +/// Does not modify the worktree. +/// +/// This should generate the same commit IDs we would have loaded in [`apply_patches`]. +/// +/// This is required to avoid commit ID churn because of author/committer mismatch +/// (generated whenever a commit is modified after the initial commit), and +/// whitespace mangling in git-format-patch. +#[tracing::instrument(skip(repo))] +pub fn canonicalize_commit_history(repo: &Repository, base_commit: Oid, leaf_commit: Oid) -> Oid { + tracing::info!("canonicalizing commit history"); + let mut canonicalize_revwalk = repo.revwalk().unwrap(); + canonicalize_revwalk.push(leaf_commit).unwrap(); + canonicalize_revwalk.hide(base_commit).unwrap(); + canonicalize_revwalk + .set_sorting(git2::Sort::TOPOLOGICAL | git2::Sort::REVERSE) + .unwrap(); + let mut last_canonical_commit = base_commit; + for original in canonicalize_revwalk { + let original = repo.find_commit(original.unwrap()).unwrap(); + let author = original.author(); + last_canonical_commit = repo + .commit( + None, + &author, + &author, + original.message().unwrap().trim(), + &original.tree().unwrap(), + &[&repo.find_commit(last_canonical_commit).unwrap()], + ) + .unwrap(); + } + tracing::info!( + leaf_commit.canonical = %last_canonical_commit, + "canonicalized commit history" + ); + last_canonical_commit +} + +/// Formats the commits between `base_commit` (exclusive) and `leaf_commit` (inclusive) as patches in `patch_dir`. +/// +/// Deletes any existing patch files in `patch_dir`. +#[tracing::instrument(skip(repo))] +pub fn format_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid, leaf_commit: Oid) { + tracing::info!("deleting existing patch files"); + // git format-patch is happy to overwrite existing files, + // but we also want to delete removed (or renamed) patch files. + for entry in patch_dir.read_dir().unwrap() { + let path = entry.unwrap().path(); + // git format-patch emits the mailbox format, not stgit(/quilt), + // so also remove markers that make it look like that + if path.file_name().is_some_and(|x| x == "series") + || path.extension().is_some_and(|x| x == "patch") + { + std::fs::remove_file(path).unwrap(); + } + } + + tracing::info!("exporting commits since base"); + if !raw_git_cmd(repo) + .arg("format-patch") + .arg(format!("{base_commit}..{leaf_commit}")) + .arg("-o") + .arg(patch_dir) + .arg("--keep-subject") + .status() + .unwrap() + .success() + { + panic!("failed to format patches"); + } +} diff --git a/patchable/src/repo.rs b/patchable/src/repo.rs new file mode 100644 index 000000000..5485aa0ec --- /dev/null +++ b/patchable/src/repo.rs @@ -0,0 +1,110 @@ +use std::path::Path; + +use git2::{FetchOptions, ObjectType, Oid, Repository, RepositoryInitOptions, WorktreeAddOptions}; + +/// Open the Git repository at `path`, creating it if it doesn't already exist. +#[tracing::instrument] +pub fn ensure_bare_repo(path: &Path) -> Repository { + match Repository::open(path) { + Ok(repo) => { + tracing::info!("repository found, reusing"); + repo + } + Err(err) => { + tracing::info!( + error = &err as &dyn std::error::Error, + "repository not found, initializing" + ); + let repo = Repository::init_opts( + path, + RepositoryInitOptions::new() + .bare(true) + .external_template(false), + ) + .unwrap(); + + repo + } + } +} + +/// Pull `commit` from `upstream_url`, if it doesn't already exist. +#[tracing::instrument(skip(repo))] +pub fn ensure_commit_exists_or_pull(repo: &Repository, commit: &str, upstream_url: &str) -> Oid { + match repo.revparse_single(commit) { + Ok(commit_obj) => { + tracing::info!("base commit exists, reusing"); + commit_obj + } + Err(err) => { + tracing::info!( + error = &err as &dyn std::error::Error, + "base commit not found, fetching from upstream" + ); + repo.remote_anonymous(upstream_url) + .unwrap() + .fetch( + &[commit], + Some( + FetchOptions::new() + // TODO: could be 1, CLI option maybe? + .depth(0), + ), + Some("fetching patchable base commit"), + ) + .unwrap(); + tracing::info!("fetched base commit"); + repo.revparse_single(commit).unwrap() + } + } + .peel_to_commit() + .unwrap() + .id() +} + +/// Ensure that the worktree at `worktree_root` exists and is checked out at `branch`. +/// +/// The worktree will be created if necessary, and the branch will be created or reset to `commit`. +#[tracing::instrument(skip(repo))] +pub fn ensure_worktree_is_at( + repo: &Repository, + worktree_name: &str, + worktree_root: &Path, + branch: &str, + commit: Oid, +) { + match Repository::open(worktree_root) { + Ok(worktree) => { + tracing::info!("worktree found, reusing and resetting"); + // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime + worktree + .set_head_detached(worktree.head().unwrap().peel_to_commit().unwrap().id()) + .unwrap(); + let branch = worktree + .branch(branch, &worktree.find_commit(commit).unwrap(), true) + .unwrap() + .into_reference(); + worktree + .checkout_tree(&branch.peel(ObjectType::Commit).unwrap(), None) + .unwrap(); + worktree.set_head_bytes(branch.name_bytes()).unwrap(); + } + Err(err) => { + tracing::info!( + error = &err as &dyn std::error::Error, + "worktree not found, creating" + ); + std::fs::create_dir_all(worktree_root.parent().unwrap()).unwrap(); + let worktree_ref = repo + .branch(branch, &repo.find_commit(commit).unwrap(), true) + .unwrap() + .into_reference(); + repo.worktree( + worktree_name, + worktree_root, + Some(WorktreeAddOptions::new().reference(Some(&worktree_ref))), + ) + .unwrap(); + } + } +} diff --git a/patchable/src/utils.rs b/patchable/src/utils.rs new file mode 100644 index 000000000..7a165484f --- /dev/null +++ b/patchable/src/utils.rs @@ -0,0 +1,16 @@ +use std::path::Path; + +use git2::Repository; + +/// Runs a raw git command in the environment of a Git repository. +/// +/// Used for functionality that is not currently implemented by libgit2/gix. +pub fn raw_git_cmd(repo: &Repository) -> std::process::Command { + let mut cmd = std::process::Command::new("git"); + cmd.env("GIT_DIR", repo.path()); + cmd.env( + "GIT_WORK_TREE", + repo.workdir().unwrap_or(Path::new("/dev/null")), + ); + cmd +} From 6741b2befe37cac8eb9d32638ba8d5397fcf2165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 7 Feb 2025 11:59:55 +0100 Subject: [PATCH 22/48] Start handling errors --- Cargo.lock | 22 +++++ Cargo.toml | 1 + patchable/Cargo.toml | 1 + patchable/src/error.rs | 82 ++++++++++++++++ patchable/src/main.rs | 36 +++++-- patchable/src/repo.rs | 211 ++++++++++++++++++++++++++++++++++------- 6 files changed, 313 insertions(+), 40 deletions(-) create mode 100644 patchable/src/error.rs diff --git a/Cargo.lock b/Cargo.lock index e33480770..71183abcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -522,6 +522,7 @@ dependencies = [ "clap", "git2", "serde", + "snafu", "tempfile", "time", "toml", @@ -678,6 +679,27 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "snafu" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index a475ead32..47aa07586 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ resolver = "2" clap = { version = "4.5.27", features = ["derive"] } git2 = "0.20.0" serde = { version = "1.0.217", features = ["derive"] } +snafu = "0.8.5" tempfile = "3.16.0" time = { version = "0.3.37", features = ["parsing"] } toml = "0.8.19" diff --git a/patchable/Cargo.toml b/patchable/Cargo.toml index 9874acb1a..69895307c 100644 --- a/patchable/Cargo.toml +++ b/patchable/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" clap.workspace = true git2.workspace = true serde.workspace = true +snafu.workspace = true tempfile.workspace = true time.workspace = true toml.workspace = true diff --git a/patchable/src/error.rs b/patchable/src/error.rs new file mode 100644 index 000000000..1d15f308a --- /dev/null +++ b/patchable/src/error.rs @@ -0,0 +1,82 @@ +//! Error type helpers. + +use std::{ + fmt::Display, + path::{Path, PathBuf}, +}; + +use git2::{Object, Oid, Reference, Repository}; + +#[derive(Debug)] +pub struct CommitId(Oid); +impl Display for CommitId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} +impl From for CommitId { + fn from(value: Oid) -> Self { + Self(value) + } +} +impl From<&Object<'_>> for CommitId { + fn from(value: &Object<'_>) -> Self { + value.id().into() + } +} +impl From> for CommitId { + fn from(value: Object<'_>) -> Self { + (&value).into() + } +} + +#[derive(Debug)] +pub struct CommitRef(String); +impl Display for CommitRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } +} +impl From<&str> for CommitRef { + fn from(value: &str) -> Self { + Self(value.into()) + } +} +impl From for CommitRef { + fn from(value: Oid) -> Self { + Self(value.to_string()) + } +} +impl From> for CommitRef { + fn from(value: Object<'_>) -> Self { + value.id().into() + } +} +impl From<&Reference<'_>> for CommitRef { + fn from(value: &Reference<'_>) -> Self { + value.name().unwrap_or("").into() + } +} +impl From> for CommitRef { + fn from(value: Reference<'_>) -> Self { + (&value).into() + } +} + +#[derive(Debug)] +pub struct RepoPath(PathBuf); +impl Display for RepoPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } +} +impl From<&Path> for RepoPath { + fn from(value: &Path) -> Self { + Self(value.into()) + } +} +impl From<&Repository> for RepoPath { + fn from(value: &Repository) -> Self { + value.path().into() + } +} diff --git a/patchable/src/main.rs b/patchable/src/main.rs index bf6364d3e..e1d6957ae 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -1,3 +1,4 @@ +mod error; mod patch; mod repo; mod utils; @@ -7,6 +8,7 @@ use std::path::{Path, PathBuf}; use git2::Repository; use serde::Deserialize; +use snafu::{OptionExt, ResultExt as _, Snafu}; use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; #[derive(clap::Parser)] @@ -85,7 +87,24 @@ enum Cmd { }, } -fn main() { +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to find images repository"))] + FindImagesRepo { source: git2::Error }, + #[snafu(display("images repository has no work directory"))] + NoImagesRepoWorkdir, + + #[snafu(display("failed to fetch patch series' base commit"))] + FetchBaseCommit { source: repo::Error }, + + #[snafu(display("failed to open product repository"))] + OpenProductRepoForCheckout { source: repo::Error }, + #[snafu(display("failed to checkout product worktree"))] + CheckoutProductWorktree { source: repo::Error }, +} + +#[snafu::report] +fn main() -> Result<(), Error> { tracing_subscriber::registry() .with(tracing_subscriber::fmt::layer()) .with( @@ -109,8 +128,8 @@ fn main() { .unwrap(); let opts = ::parse(); - let images_repo = Repository::discover(".").unwrap(); - let images_repo_root = images_repo.workdir().unwrap(); + let images_repo = Repository::discover(".").context(FindImagesRepoSnafu)?; + let images_repo_root = images_repo.workdir().context(NoImagesRepoWorkdirSnafu)?; match opts.cmd { Cmd::Checkout { pv } => { let ctx = ProductVersionContext { @@ -123,10 +142,12 @@ fn main() { "finding product repository", product.repository = ?product_repo_root, ) - .in_scope(|| repo::ensure_bare_repo(&product_repo_root)); + .in_scope(|| repo::ensure_bare_repo(&product_repo_root)) + .context(OpenProductRepoForCheckoutSnafu)?; let base_commit = - repo::ensure_commit_exists_or_pull(&product_repo, &config.base, &config.upstream); + repo::ensure_commit_exists_or_fetch(&product_repo, &config.base, &config.upstream) + .context(FetchBaseCommitSnafu)?; let patched_commit = patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit); let product_worktree_root = ctx.worktree_root(); @@ -136,7 +157,8 @@ fn main() { &product_worktree_root, &ctx.worktree_branch(), patched_commit, - ); + ) + .context(CheckoutProductWorktreeSnafu)?; tracing::info!( worktree.root = ?product_worktree_root, @@ -188,4 +210,6 @@ fn main() { ); } } + + Ok(()) } diff --git a/patchable/src/repo.rs b/patchable/src/repo.rs index 5485aa0ec..840763578 100644 --- a/patchable/src/repo.rs +++ b/patchable/src/repo.rs @@ -1,48 +1,129 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use git2::{FetchOptions, ObjectType, Oid, Repository, RepositoryInitOptions, WorktreeAddOptions}; +use snafu::{ResultExt, Snafu}; + +use crate::error::{self, CommitRef}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to init repository at {path:?}"))] + Init { source: git2::Error, path: PathBuf }, + #[snafu(display("failed to open repository at {path:?}"))] + Open { source: git2::Error, path: PathBuf }, + + #[snafu(display( + "failed to create worktree branch {branch:?} pointing at {commit} in {repo}" + ))] + CreateWorktreeBranch { + source: git2::Error, + repo: error::RepoPath, + branch: String, + commit: error::CommitId, + }, + #[snafu(display("failed to create worktree folder at {path:?}"))] + CreateWorktreePath { + source: std::io::Error, + path: PathBuf, + }, + #[snafu(display("failed to create worktree {path:?} pointing at {branch} in {repo}"))] + CreateWorktree { + source: git2::Error, + repo: error::RepoPath, + path: PathBuf, + branch: CommitRef, + }, + + #[snafu(display("failed to detach worktree {worktree} from {old_target} to {commit}"))] + DetachWorktree { + source: git2::Error, + worktree: error::RepoPath, + old_target: error::CommitRef, + commit: error::CommitId, + }, + #[snafu(display("failed to checkout commit {commit} to worktree {worktree}"))] + CheckoutWorktree { + source: git2::Error, + worktree: error::RepoPath, + commit: error::CommitId, + }, + #[snafu(display("failed to update worktree {worktree}'s head to {target}"))] + UpdateWorktreeHead { + source: git2::Error, + worktree: error::RepoPath, + target: error::CommitRef, + }, + + #[snafu(display("failed to find {commit} in {repo}"))] + FindCommit { + source: git2::Error, + repo: error::RepoPath, + commit: error::CommitRef, + }, + + #[snafu(display("failed to create remote in {repo} for {url:?}"))] + CreateRemote { + source: git2::Error, + repo: error::RepoPath, + url: String, + }, + #[snafu(display("failed to fetch refs {refs:?} from {url:?} to {repo}"))] + Fetch { + source: git2::Error, + repo: error::RepoPath, + url: String, + refs: Vec, + }, +} +type Result = std::result::Result; /// Open the Git repository at `path`, creating it if it doesn't already exist. #[tracing::instrument] -pub fn ensure_bare_repo(path: &Path) -> Repository { +pub fn ensure_bare_repo(path: &Path) -> Result { match Repository::open(path) { Ok(repo) => { tracing::info!("repository found, reusing"); - repo + Ok(repo) } - Err(err) => { + Err(err) if err.code() == git2::ErrorCode::NotFound => { tracing::info!( error = &err as &dyn std::error::Error, "repository not found, initializing" ); - let repo = Repository::init_opts( + Repository::init_opts( path, RepositoryInitOptions::new() .bare(true) .external_template(false), ) - .unwrap(); - - repo + .context(InitSnafu { path }) } + Err(err) => Err(err).context(OpenSnafu { path }), } } -/// Pull `commit` from `upstream_url`, if it doesn't already exist. +/// Fetch `commit` from `upstream_url`, if it doesn't already exist. #[tracing::instrument(skip(repo))] -pub fn ensure_commit_exists_or_pull(repo: &Repository, commit: &str, upstream_url: &str) -> Oid { - match repo.revparse_single(commit) { +pub fn ensure_commit_exists_or_fetch( + repo: &Repository, + commit: &str, + upstream_url: &str, +) -> Result { + let commit = match repo.revparse_single(commit) { Ok(commit_obj) => { tracing::info!("base commit exists, reusing"); - commit_obj + Ok(commit_obj) } - Err(err) => { + Err(err) if err.code() == git2::ErrorCode::NotFound => { tracing::info!( error = &err as &dyn std::error::Error, "base commit not found, fetching from upstream" ); repo.remote_anonymous(upstream_url) - .unwrap() + .context(CreateRemoteSnafu { + repo, + url: upstream_url, + })? .fetch( &[commit], Some( @@ -52,14 +133,21 @@ pub fn ensure_commit_exists_or_pull(repo: &Repository, commit: &str, upstream_ur ), Some("fetching patchable base commit"), ) - .unwrap(); + .with_context(|_| FetchSnafu { + repo, + url: upstream_url, + refs: vec![commit.to_string()], + })?; tracing::info!("fetched base commit"); - repo.revparse_single(commit).unwrap() + repo.revparse_single(commit) } + Err(err) => Err(err), } - .peel_to_commit() - .unwrap() - .id() + .context(FindCommitSnafu { repo, commit })?; + Ok(commit + .peel_to_commit() + .context(FindCommitSnafu { repo, commit })? + .id()) } /// Ensure that the worktree at `worktree_root` exists and is checked out at `branch`. @@ -72,39 +160,94 @@ pub fn ensure_worktree_is_at( worktree_root: &Path, branch: &str, commit: Oid, -) { +) -> Result<()> { + tracing::info!("checking out worktree"); + let commit_obj = repo + .find_commit(commit) + .context(FindCommitSnafu { repo, commit })?; match Repository::open(worktree_root) { Ok(worktree) => { - tracing::info!("worktree found, reusing and resetting"); + tracing::info!("worktree found, reusing"); // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime - worktree - .set_head_detached(worktree.head().unwrap().peel_to_commit().unwrap().id()) - .unwrap(); + if let Ok(head) = worktree.head() { + tracing::info!(head.old = head.name(), "detaching worktree head"); + let head_commit = head + .peel_to_commit() + .context(FindCommitSnafu { + repo: &worktree, + commit: &head, + })? + .id(); + worktree + .set_head_detached(head_commit) + .context(DetachWorktreeSnafu { + worktree: worktree_root, + old_target: head, + commit: head_commit, + })?; + } let branch = worktree - .branch(branch, &worktree.find_commit(commit).unwrap(), true) - .unwrap() + .branch(branch, &commit_obj, true) + .context(CreateWorktreeBranchSnafu { + repo: &worktree, + branch, + commit, + })? .into_reference(); + let commit = branch.peel(ObjectType::Commit).context(FindCommitSnafu { + repo: &worktree, + commit: &branch, + })?; + worktree + .checkout_tree(&commit, None) + .context(CheckoutWorktreeSnafu { + worktree: &worktree, + commit: &commit, + })?; worktree - .checkout_tree(&branch.peel(ObjectType::Commit).unwrap(), None) - .unwrap(); - worktree.set_head_bytes(branch.name_bytes()).unwrap(); + .set_head_bytes(branch.name_bytes()) + .context(UpdateWorktreeHeadSnafu { + worktree: &worktree, + target: &branch, + })?; + Ok(()) } - Err(err) => { + Err(err) if err.code() == git2::ErrorCode::NotFound => { tracing::info!( error = &err as &dyn std::error::Error, "worktree not found, creating" ); - std::fs::create_dir_all(worktree_root.parent().unwrap()).unwrap(); + std::fs::create_dir_all(worktree_root).context(CreateWorktreePathSnafu { + path: worktree_root, + })?; let worktree_ref = repo - .branch(branch, &repo.find_commit(commit).unwrap(), true) - .unwrap() + .branch( + branch, + &repo + .find_commit(commit) + .context(FindCommitSnafu { repo, commit })?, + true, + ) + .context(CreateWorktreeBranchSnafu { + repo, + branch, + commit, + })? .into_reference(); repo.worktree( worktree_name, worktree_root, Some(WorktreeAddOptions::new().reference(Some(&worktree_ref))), ) - .unwrap(); + .context(CreateWorktreeSnafu { + repo, + path: worktree_root, + branch: worktree_ref, + })?; + Ok(()) } + Err(err) => Err(err).context(OpenSnafu { + path: worktree_root, + }), } } From 7d82e156f88e5c9ea28a0b87f67ba0478c1f9684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 7 Feb 2025 13:30:18 +0100 Subject: [PATCH 23/48] Factor out patch mail parsing --- patchable/src/main.rs | 4 +- patchable/src/patch.rs | 185 ++++++++++++++---------------------- patchable/src/patch_mail.rs | 119 +++++++++++++++++++++++ patchable/src/repo.rs | 13 +-- 4 files changed, 202 insertions(+), 119 deletions(-) create mode 100644 patchable/src/patch_mail.rs diff --git a/patchable/src/main.rs b/patchable/src/main.rs index e1d6957ae..97caff4b5 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -1,5 +1,6 @@ mod error; mod patch; +mod patch_mail; mod repo; mod utils; @@ -148,7 +149,8 @@ fn main() -> Result<(), Error> { let base_commit = repo::ensure_commit_exists_or_fetch(&product_repo, &config.base, &config.upstream) .context(FetchBaseCommitSnafu)?; - let patched_commit = patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit); + let patched_commit = + patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit).unwrap(); let product_worktree_root = ctx.worktree_root(); repo::ensure_worktree_is_at( diff --git a/patchable/src/patch.rs b/patchable/src/patch.rs index f44b42f55..a8bdb906e 100644 --- a/patchable/src/patch.rs +++ b/patchable/src/patch.rs @@ -1,10 +1,65 @@ -use std::{fs::File, path::Path, process::Stdio}; +use std::path::{Path, PathBuf}; -use git2::{Diff, Oid, Repository, Signature}; -use tempfile::tempdir; -use time::{format_description::well_known::Rfc2822, OffsetDateTime}; +use git2::{Oid, Repository}; +use snafu::{ResultExt as _, Snafu}; -use crate::utils::raw_git_cmd; +use crate::{ + patch_mail::{mailinfo, mailsplit}, + utils::raw_git_cmd, +}; + +#[derive(Debug, Snafu)] +pub enum Error { + OpenStgitSeriesFile { + source: std::io::Error, + path: PathBuf, + }, + ListPatchDirectory { + source: std::io::Error, + path: PathBuf, + }, +} +type Result = std::result::Result; + +/// Lists the patches to apply in `patch_dir`, in order. +fn patch_files(patch_dir: &Path) -> Result> { + let series_file = patch_dir.join("series"); + Ok(match std::fs::read_to_string(&series_file) { + Ok(file) => { + tracing::info!( + patch.series = %series_file.display(), + "series file found, treating as stgit series" + ); + file.lines() + .map(|file_name| patch_dir.join(file_name)) + .collect::>() + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + tracing::info!( + error = &err as &dyn std::error::Error, + patch.series = %series_file.display(), + "series file not found, treating as git mailbox" + ); + let mut patch_files = patch_dir + .read_dir() + .unwrap() + .filter_map(|e| { + e.map(|entry| { + let path = entry.path(); + path.extension() + .is_some_and(|ext| ext == "patch") + .then_some(path) + }) + .transpose() + }) + .collect::, _>>() + .context(ListPatchDirectorySnafu { path: patch_dir })?; + patch_files.sort(); + patch_files + } + Err(err) => return Err(err).context(OpenStgitSeriesFileSnafu { path: series_file }), + }) +} /// Apply the patches in `patch_dir` to `base_commit`. /// @@ -16,130 +71,36 @@ use crate::utils::raw_git_cmd; /// (letting us keep any dirty files in the worktree that don't conflict with the final switcheroo, /// even if those files are modified by some of the patches) #[tracing::instrument(skip(repo))] -pub fn apply_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid) -> Oid { +pub fn apply_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid) -> Result { tracing::info!( patch.dir = %patch_dir.display(), "applying patches" ); - let series_file = patch_dir.join("series"); - let patch_files = if series_file.exists() { - tracing::info!( - patch.series = %series_file.display(), - "series file found, treating as stgit series" - ); - std::fs::read_to_string(series_file) - .unwrap() - .lines() - .map(|file_name| patch_dir.join(file_name)) - .collect::>() - } else { - tracing::info!( - patch.series = %series_file.display(), - "series file not found, treating as git mailbox" - ); - let mut patch_files = patch_dir - .read_dir() - .unwrap() - .map(|x| x.unwrap().path()) - .filter(|x| x.extension().is_some_and(|x| x == "patch")) - .collect::>(); - patch_files.sort(); - patch_files - }; let mut last_commit_id = base_commit; - for patch_file in patch_files { + for patch_file in patch_files(patch_dir)? { tracing::info!( - patch.file = %patch_file.display(), + patch.file = ?patch_file, "parsing patch" ); - let mailsplit_dir = tempdir().unwrap(); - let mailsplit = raw_git_cmd(repo) - .arg("mailsplit") - // mailsplit doesn't accept split arguments ("-o dir") - .arg(format!("-o{}", mailsplit_dir.path().to_str().unwrap())) - .arg("--") - .arg(patch_file) - .stderr(Stdio::inherit()) - .output() - .unwrap(); - if !mailsplit.status.success() { - panic!("failed to apply patches"); - } - let mailsplit_patch_count = std::str::from_utf8(&mailsplit.stdout) - .unwrap() - .trim() - .parse::() - .unwrap(); - for patch_i in 1..=mailsplit_patch_count { - // Matches the format emitted by git-mailsplit - let patch_mail_file_name = format!("{patch_i:04}"); - let patch_split_msg_file = mailsplit_dir - .path() - .join(format!("{patch_mail_file_name}-msg")); - let patch_split_patch_file = mailsplit_dir - .path() - .join(format!("{patch_mail_file_name}-patch")); - let mailinfo = raw_git_cmd(repo) - .arg("mailinfo") - .args([&patch_split_msg_file, &patch_split_patch_file]) - .stdin(File::open(mailsplit_dir.path().join(patch_mail_file_name)).unwrap()) - .stderr(Stdio::inherit()) - .output() - .unwrap(); - if !mailinfo.status.success() { - panic!("failed to apply patches"); - } - let patch_info = std::str::from_utf8(&mailinfo.stdout).unwrap(); - - let mut author_name = None; - let mut author_email = None; - let mut date = None; - let mut subject = None; - for patch_info_line in patch_info.lines() { - if !patch_info_line.is_empty() { - match patch_info_line.split_once(": ").unwrap() { - ("Author", x) => author_name = Some(x), - ("Email", x) => author_email = Some(x), - ("Date", x) => date = Some(x), - ("Subject", x) => subject = Some(x), - (header, _) => panic!("unknown header type {header:?}"), - } - } - } - let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); - let author = Signature::new( - author_name.unwrap(), - author_email.unwrap(), - &git2::Time::new(date.unix_timestamp(), date.offset().whole_minutes().into()), - ) - .unwrap(); - let parent_commit = repo.find_commit(last_commit_id).unwrap(); + for patch_email_file in mailsplit(repo, &patch_file).unwrap() { + let patch = mailinfo(repo, &patch_email_file).unwrap().parse().unwrap(); tracing::info!( - commit.base = %parent_commit.id(), - commit.subject = subject.unwrap(), + commit.base = %last_commit_id, + commit.subject = patch.subject, "applying patch" ); + let parent_commit = repo.find_commit(last_commit_id).unwrap(); let patch_tree_id = repo - .apply_to_tree( - &parent_commit.tree().unwrap(), - &Diff::from_buffer(&std::fs::read(patch_split_patch_file).unwrap()).unwrap(), - None, - ) + .apply_to_tree(&parent_commit.tree().unwrap(), &patch.patch, None) .unwrap() .write_tree_to(repo) .unwrap(); - let msg = std::fs::read_to_string(patch_split_msg_file).unwrap(); - let full_msg = if msg.is_empty() { - subject.unwrap() - } else { - &format!("{}\n\n{msg}", subject.unwrap()) - }; last_commit_id = repo .commit( None, - &author, - &author, - full_msg.trim(), + &patch.author, + &patch.author, + &patch.message, &repo.find_tree(patch_tree_id).unwrap(), &[&parent_commit], ) @@ -150,7 +111,7 @@ pub fn apply_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid) -> O ); } } - last_commit_id + Ok(last_commit_id) } /// Canonicalize commits for all commits between `base_commit` (exclusive) and `leaf_commit` (inclusive). diff --git a/patchable/src/patch_mail.rs b/patchable/src/patch_mail.rs new file mode 100644 index 000000000..0898c406a --- /dev/null +++ b/patchable/src/patch_mail.rs @@ -0,0 +1,119 @@ +use std::{ + fs::File, + path::{Path, PathBuf}, + process::Stdio, +}; + +use git2::{Diff, Repository, Signature}; +use snafu::Snafu; +use tempfile::{tempdir, NamedTempFile}; +use time::{format_description::well_known::Rfc2822, OffsetDateTime}; + +use crate::utils::raw_git_cmd; + +#[derive(Debug, Snafu)] +pub enum Error {} +type Result = std::result::Result; + +/// Splits a series of git patch emails into individual patch emails. +pub fn mailsplit(repo: &Repository, patch_file: &Path) -> Result> { + let base_dir = tempdir().unwrap(); + let mailsplit = raw_git_cmd(repo) + .arg("mailsplit") + // mailsplit doesn't accept split arguments ("-o dir") + .arg(format!("-o{}", base_dir.path().to_str().unwrap())) + .arg("--") + .arg(patch_file) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + if !mailsplit.status.success() { + panic!("failed to apply patches"); + } + let mailsplit_patch_count = std::str::from_utf8(&mailsplit.stdout) + .unwrap() + .trim() + .parse() + .unwrap(); + Ok((1..=mailsplit_patch_count).map(move |patch_i| { + base_dir.path().join( + // Matches the format emitted by git-mailsplit + format!("{patch_i:04}"), + ) + })) +} + +pub struct Mailinfo { + headers: String, + rest_of_message: String, + patch: Vec, +} + +pub struct ParsedPatch { + pub subject: String, + pub message: String, + pub author: Signature<'static>, + pub patch: Diff<'static>, +} + +impl Mailinfo { + pub fn parse(self) -> Result { + let mut author_name = None; + let mut author_email = None; + let mut date = None; + let mut subject = None; + for patch_info_line in self.headers.lines() { + if !patch_info_line.is_empty() { + match patch_info_line.split_once(": ").unwrap() { + ("Author", x) => author_name = Some(x), + ("Email", x) => author_email = Some(x), + ("Date", x) => date = Some(x), + ("Subject", x) => subject = Some(x), + (header, _) => panic!("unknown header type {header:?}"), + } + } + } + let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); + let full_msg = if self.rest_of_message.is_empty() { + subject.unwrap().trim().to_string() + } else { + format!( + "{}\n\n{}", + subject.unwrap().trim(), + self.rest_of_message.trim() + ) + }; + Ok(ParsedPatch { + subject: subject.unwrap().trim().to_string(), + message: full_msg, + author: Signature::new( + author_name.unwrap(), + author_email.unwrap(), + &git2::Time::new(date.unix_timestamp(), date.offset().whole_minutes().into()), + ) + .unwrap(), + patch: Diff::from_buffer(&self.patch).unwrap(), + }) + } +} + +pub fn mailinfo(repo: &Repository, patch_email_file: &Path) -> Result { + let msg_file = NamedTempFile::new().unwrap().into_temp_path(); + let patch_file = NamedTempFile::new().unwrap().into_temp_path(); + let mailinfo = raw_git_cmd(repo) + .arg("mailinfo") + .args([&msg_file, &patch_file]) + .stdin(File::open(patch_email_file).unwrap()) + .stderr(Stdio::inherit()) + .output() + .unwrap(); + if !mailinfo.status.success() { + panic!("failed to apply patches"); + } + let patch_info = std::str::from_utf8(&mailinfo.stdout).unwrap(); + Ok(Mailinfo { + headers: patch_info.to_string(), + rest_of_message: std::fs::read_to_string(msg_file).unwrap(), + patch: std::fs::read(patch_file).unwrap(), + }) +} diff --git a/patchable/src/repo.rs b/patchable/src/repo.rs index 840763578..fe3cfd7b4 100644 --- a/patchable/src/repo.rs +++ b/patchable/src/repo.rs @@ -162,12 +162,12 @@ pub fn ensure_worktree_is_at( commit: Oid, ) -> Result<()> { tracing::info!("checking out worktree"); - let commit_obj = repo - .find_commit(commit) - .context(FindCommitSnafu { repo, commit })?; match Repository::open(worktree_root) { Ok(worktree) => { tracing::info!("worktree found, reusing"); + let commit_obj = worktree + .find_commit(commit) + .context(FindCommitSnafu { repo, commit })?; // We can't reset the branch if it's already checked out, so detach to the commit instead for the meantime if let Ok(head) = worktree.head() { tracing::info!(head.old = head.name(), "detaching worktree head"); @@ -217,9 +217,10 @@ pub fn ensure_worktree_is_at( error = &err as &dyn std::error::Error, "worktree not found, creating" ); - std::fs::create_dir_all(worktree_root).context(CreateWorktreePathSnafu { - path: worktree_root, - })?; + if let Some(parent) = worktree_root.parent() { + std::fs::create_dir_all(parent) + .context(CreateWorktreePathSnafu { path: parent })?; + } let worktree_ref = repo .branch( branch, From 590ca017ba301cc689061fc65e0343a7cba019df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 7 Feb 2025 14:59:35 +0100 Subject: [PATCH 24/48] More error handling --- patchable/src/error.rs | 7 +- patchable/src/main.rs | 6 +- patchable/src/patch.rs | 260 +++++++++++++++++++++++++++++------- patchable/src/patch_mail.rs | 136 ++++++++++++++----- 4 files changed, 330 insertions(+), 79 deletions(-) diff --git a/patchable/src/error.rs b/patchable/src/error.rs index 1d15f308a..d0df4fa69 100644 --- a/patchable/src/error.rs +++ b/patchable/src/error.rs @@ -5,7 +5,7 @@ use std::{ path::{Path, PathBuf}, }; -use git2::{Object, Oid, Reference, Repository}; +use git2::{Commit, Object, Oid, Reference, Repository}; #[derive(Debug)] pub struct CommitId(Oid); @@ -19,6 +19,11 @@ impl From for CommitId { Self(value) } } +impl From<&Commit<'_>> for CommitId { + fn from(value: &Commit<'_>) -> Self { + value.id().into() + } +} impl From<&Object<'_>> for CommitId { fn from(value: &Object<'_>) -> Self { value.id().into() diff --git a/patchable/src/main.rs b/patchable/src/main.rs index 97caff4b5..5e43d4db1 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -196,7 +196,8 @@ fn main() -> Result<(), Error> { .peel_to_commit() .unwrap() .id(), - ); + ) + .unwrap(); let patch_dir = ctx.patch_dir(); patch::format_patches( @@ -204,7 +205,8 @@ fn main() -> Result<(), Error> { &patch_dir, base_commit, canonical_leaf_commit, - ); + ) + .unwrap(); tracing::info!( patch.dir = ?patch_dir, diff --git a/patchable/src/patch.rs b/patchable/src/patch.rs index a8bdb906e..5e685470e 100644 --- a/patchable/src/patch.rs +++ b/patchable/src/patch.rs @@ -1,23 +1,120 @@ -use std::path::{Path, PathBuf}; +use std::{ + path::{Path, PathBuf}, + process::ExitStatus, +}; use git2::{Oid, Repository}; -use snafu::{ResultExt as _, Snafu}; +use snafu::{OptionExt, ResultExt as _, Snafu}; use crate::{ - patch_mail::{mailinfo, mailsplit}, + error::{self, CommitId}, + patch_mail::{self, mailinfo, mailsplit}, utils::raw_git_cmd, }; #[derive(Debug, Snafu)] pub enum Error { + #[snafu(display("failed to open stgit series file {path:?}"))] OpenStgitSeriesFile { source: std::io::Error, path: PathBuf, }, + #[snafu(display("failed to list contents of patch directory {path:?}"))] ListPatchDirectory { source: std::io::Error, path: PathBuf, }, + + #[snafu(display("failed to split patch file {patch_file:?}"))] + Mailsplit { + source: patch_mail::Error, + patch_file: PathBuf, + }, + #[snafu(display( + "failed to split patch email file {patch_email_file:?} (from {patch_file:?})" + ))] + Mailinfo { + source: patch_mail::Error, + patch_email_file: PathBuf, + patch_file: PathBuf, + }, + #[snafu(display( + "failed to parse patch email file {patch_email_file:?} (from {patch_file:?})" + ))] + ParseMailinfo { + source: patch_mail::Error, + patch_email_file: PathBuf, + patch_file: PathBuf, + }, + + #[snafu(display("failed to find parent commit {commit}"))] + FindParentCommit { + source: git2::Error, + commit: error::CommitId, + }, + #[snafu(display("failed to read tree of parent commit {parent_commit}"))] + ReadParentCommitTree { + source: git2::Error, + parent_commit: error::CommitId, + }, + #[snafu(display("failed to apply patch {patch_email_file:?} (from {patch_file:?}) to parent commit {parent_commit}"))] + ApplyPatch { + source: git2::Error, + parent_commit: error::CommitId, + patch_email_file: PathBuf, + patch_file: PathBuf, + }, + #[snafu(display("failed to write tree for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}"))] + WritePatchedTree { + source: git2::Error, + parent_commit: error::CommitId, + patch_email_file: PathBuf, + patch_file: PathBuf, + }, + #[snafu(display("failed to read patched tree {tree}"))] + ReadPatchedTree { source: git2::Error, tree: Oid }, + #[snafu(display("failed to write commit for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}"))] + WriteCommit { + source: git2::Error, + parent_commit: error::CommitId, + patch_email_file: PathBuf, + patch_file: PathBuf, + }, + + #[snafu(display("failed to configure revwalk for canonicalization"))] + ConfigureCanonicalizeRevwalk { source: git2::Error }, + #[snafu(display("revwalk returned invalid object"))] + CanonicalizeRevwalkObject { source: git2::Error }, + #[snafu(display("failed to find commit {commit} from canonicalization revwalk"))] + CanonicalizeRevwalkFindCommit { + source: git2::Error, + commit: CommitId, + }, + #[snafu(display("failed to find read tree from original commit {commit}"))] + CanonicalizeReadOriginalCommitTree { + source: git2::Error, + commit: CommitId, + }, + #[snafu(display( + "failed to write canonicalized commit of {original_commit} (with canonicalized parent {parent_commit})" + ))] + CanonicalizeWriteCommit { + source: git2::Error, + parent_commit: error::CommitId, + original_commit: error::CommitId, + }, + #[snafu(display("commit {commit}'s commit message is invalid UTF-8"))] + NonUtf8CommitMessage { commit: CommitId }, + + #[snafu(display("failed to delete old patch file {path:?}"))] + DeleteOldPatch { + source: std::io::Error, + path: PathBuf, + }, + #[snafu(display("failed to run git format-mail"))] + RunFormatMail { source: std::io::Error }, + #[snafu(display("git format-mail exited with status code {status}"))] + FormatMailFailed { status: ExitStatus }, } type Result = std::result::Result; @@ -42,17 +139,19 @@ fn patch_files(patch_dir: &Path) -> Result> { ); let mut patch_files = patch_dir .read_dir() - .unwrap() - .filter_map(|e| { - e.map(|entry| { - let path = entry.path(); - path.extension() - .is_some_and(|ext| ext == "patch") - .then_some(path) - }) - .transpose() + .and_then(|entries| { + entries + .filter_map(|e| { + e.map(|entry| { + let path = entry.path(); + path.extension() + .is_some_and(|ext| ext == "patch") + .then_some(path) + }) + .transpose() + }) + .collect::, _>>() }) - .collect::, _>>() .context(ListPatchDirectorySnafu { path: patch_dir })?; patch_files.sort(); patch_files @@ -77,34 +176,72 @@ pub fn apply_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid) -> R "applying patches" ); let mut last_commit_id = base_commit; - for patch_file in patch_files(patch_dir)? { + for ref patch_file in patch_files(patch_dir)? { tracing::info!( patch.file = ?patch_file, "parsing patch" ); - for patch_email_file in mailsplit(repo, &patch_file).unwrap() { - let patch = mailinfo(repo, &patch_email_file).unwrap().parse().unwrap(); + for ref patch_email_file in + mailsplit(repo, patch_file).context(MailsplitSnafu { patch_file })? + { + let patch = mailinfo(repo, patch_email_file) + .context(MailinfoSnafu { + patch_email_file, + patch_file, + })? + .parse() + .context(ParseMailinfoSnafu { + patch_email_file, + patch_file, + })?; tracing::info!( commit.base = %last_commit_id, commit.subject = patch.subject, "applying patch" ); - let parent_commit = repo.find_commit(last_commit_id).unwrap(); + let parent_commit = + &repo + .find_commit(last_commit_id) + .context(FindParentCommitSnafu { + commit: last_commit_id, + })?; let patch_tree_id = repo - .apply_to_tree(&parent_commit.tree().unwrap(), &patch.patch, None) - .unwrap() + .apply_to_tree( + &parent_commit + .tree() + .context(ReadParentCommitTreeSnafu { parent_commit })?, + &patch.patch, + None, + ) + .context(ApplyPatchSnafu { + parent_commit, + patch_email_file, + patch_file, + })? .write_tree_to(repo) - .unwrap(); + .context(WritePatchedTreeSnafu { + parent_commit, + patch_email_file, + patch_file, + })?; last_commit_id = repo .commit( None, &patch.author, &patch.author, &patch.message, - &repo.find_tree(patch_tree_id).unwrap(), - &[&parent_commit], + &repo + .find_tree(patch_tree_id) + .context(ReadPatchedTreeSnafu { + tree: patch_tree_id, + })?, + &[parent_commit], ) - .unwrap(); + .context(WriteCommitSnafu { + parent_commit, + patch_email_file, + patch_file, + })?; tracing::info!( commit.id = %last_commit_id, "applied patch" @@ -124,66 +261,99 @@ pub fn apply_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid) -> R /// (generated whenever a commit is modified after the initial commit), and /// whitespace mangling in git-format-patch. #[tracing::instrument(skip(repo))] -pub fn canonicalize_commit_history(repo: &Repository, base_commit: Oid, leaf_commit: Oid) -> Oid { +pub fn canonicalize_commit_history( + repo: &Repository, + base_commit: Oid, + leaf_commit: Oid, +) -> Result { tracing::info!("canonicalizing commit history"); - let mut canonicalize_revwalk = repo.revwalk().unwrap(); - canonicalize_revwalk.push(leaf_commit).unwrap(); - canonicalize_revwalk.hide(base_commit).unwrap(); - canonicalize_revwalk - .set_sorting(git2::Sort::TOPOLOGICAL | git2::Sort::REVERSE) - .unwrap(); + let canonicalize_revwalk = repo + .revwalk() + .and_then(|mut walk| { + walk.push(leaf_commit)?; + walk.hide(base_commit)?; + walk.set_sorting(git2::Sort::TOPOLOGICAL | git2::Sort::REVERSE)?; + Ok(walk) + }) + .context(ConfigureCanonicalizeRevwalkSnafu)?; let mut last_canonical_commit = base_commit; for original in canonicalize_revwalk { - let original = repo.find_commit(original.unwrap()).unwrap(); + let original = original.context(CanonicalizeRevwalkObjectSnafu)?; + let original = &repo + .find_commit(original) + .context(CanonicalizeRevwalkFindCommitSnafu { commit: original })?; let author = original.author(); last_canonical_commit = repo .commit( None, &author, &author, - original.message().unwrap().trim(), - &original.tree().unwrap(), - &[&repo.find_commit(last_canonical_commit).unwrap()], + original + .message() + .context(NonUtf8CommitMessageSnafu { commit: original })? + .trim(), + &original + .tree() + .context(CanonicalizeReadOriginalCommitTreeSnafu { commit: original })?, + &[&repo + .find_commit(last_canonical_commit) + .context(FindParentCommitSnafu { + commit: last_canonical_commit, + })?], ) - .unwrap(); + .context(CanonicalizeWriteCommitSnafu { + parent_commit: last_canonical_commit, + original_commit: original, + })?; } tracing::info!( leaf_commit.canonical = %last_canonical_commit, "canonicalized commit history" ); - last_canonical_commit + Ok(last_canonical_commit) } /// Formats the commits between `base_commit` (exclusive) and `leaf_commit` (inclusive) as patches in `patch_dir`. /// /// Deletes any existing patch files in `patch_dir`. #[tracing::instrument(skip(repo))] -pub fn format_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid, leaf_commit: Oid) { +pub fn format_patches( + repo: &Repository, + patch_dir: &Path, + base_commit: Oid, + leaf_commit: Oid, +) -> Result<()> { tracing::info!("deleting existing patch files"); // git format-patch is happy to overwrite existing files, // but we also want to delete removed (or renamed) patch files. - for entry in patch_dir.read_dir().unwrap() { - let path = entry.unwrap().path(); + for entry in patch_dir + .read_dir() + .context(ListPatchDirectorySnafu { path: patch_dir })? + { + let path = &entry + .context(ListPatchDirectorySnafu { path: patch_dir })? + .path(); // git format-patch emits the mailbox format, not stgit(/quilt), // so also remove markers that make it look like that if path.file_name().is_some_and(|x| x == "series") || path.extension().is_some_and(|x| x == "patch") { - std::fs::remove_file(path).unwrap(); + std::fs::remove_file(path).context(DeleteOldPatchSnafu { path })?; } } tracing::info!("exporting commits since base"); - if !raw_git_cmd(repo) + let status = raw_git_cmd(repo) .arg("format-patch") .arg(format!("{base_commit}..{leaf_commit}")) .arg("-o") .arg(patch_dir) .arg("--keep-subject") .status() - .unwrap() - .success() - { - panic!("failed to format patches"); + .context(RunFormatMailSnafu)?; + if !status.success() { + return FormatMailFailedSnafu { status }.fail(); } + + Ok(()) } diff --git a/patchable/src/patch_mail.rs b/patchable/src/patch_mail.rs index 0898c406a..9d120ffae 100644 --- a/patchable/src/patch_mail.rs +++ b/patchable/src/patch_mail.rs @@ -1,40 +1,104 @@ use std::{ + ffi::OsString, fs::File, + num::ParseIntError, path::{Path, PathBuf}, - process::Stdio, + process::{ExitStatus, Stdio}, + str::Utf8Error, }; use git2::{Diff, Repository, Signature}; -use snafu::Snafu; +use snafu::{OptionExt as _, ResultExt, Snafu}; use tempfile::{tempdir, NamedTempFile}; use time::{format_description::well_known::Rfc2822, OffsetDateTime}; use crate::utils::raw_git_cmd; #[derive(Debug, Snafu)] -pub enum Error {} +pub enum Error { + #[snafu(display("failed to create temporary directory"))] + CreateTempDir { source: std::io::Error }, + #[snafu(display("failed to create temporary file"))] + CreateTempFile { source: std::io::Error }, + + #[snafu(display("failed to run git mailsplit"))] + RunMailsplit { source: std::io::Error }, + #[snafu(display("git mailsplit exited with status code {status}"))] + MailsplitFailed { status: ExitStatus }, + #[snafu(display("git mailsplit returned invalid UTF-8"))] + MailsplitOutput { source: Utf8Error }, + #[snafu(display("git mailsplit invalid number"))] + ParseMailsplit { source: ParseIntError }, + + #[snafu(display( + "failed to open mail file at {path:?} (should have been created by git mailsplit)" + ))] + OpenMailFile { + source: std::io::Error, + path: PathBuf, + }, + #[snafu(display("failed to run git mailinfo"))] + RunMailinfo { source: std::io::Error }, + #[snafu(display("git mailinfo exited with status code {status}"))] + MailinfoFailed { status: ExitStatus }, + #[snafu(display("git mailsplit returned invalid UTF-8"))] + MailinfoOutput { source: Utf8Error }, + #[snafu(display("failed to read message file created by git mailinfo"))] + ReadMailinfoMessage { source: std::io::Error }, + #[snafu(display("failed to read patch file created by git mailinfo"))] + ReadMailinfoPatch { source: std::io::Error }, + + #[snafu(display("malformed mail header (should be separated by \": \")"))] + MalformedMailHeader, + #[snafu(display("unknown mail header type {header:?}"))] + UnknownMailHeader { header: String }, + #[snafu(display("patch mail has no \"Author\" header"))] + NoAuthorName, + #[snafu(display("patch mail has no \"Email\" header"))] + NoAuthorEmail, + #[snafu(display("patch mail has no \"Date\" header"))] + NoDate, + #[snafu(display("patch mail has no \"Subject\" header"))] + NoSubject, + #[snafu(display("failed to parse \"Date\" header (should be RFC2822)"))] + InvalidMailDate { + source: time::error::Parse, + date: String, + }, + #[snafu(display("failed to build commit signature from headers"))] + InvalidSignature { source: git2::Error }, + #[snafu(display("patch has invalid diff"))] + InvalidDiff { source: git2::Error }, +} type Result = std::result::Result; /// Splits a series of git patch emails into individual patch emails. pub fn mailsplit(repo: &Repository, patch_file: &Path) -> Result> { - let base_dir = tempdir().unwrap(); + let base_dir = tempdir().context(CreateTempDirSnafu)?; let mailsplit = raw_git_cmd(repo) .arg("mailsplit") // mailsplit doesn't accept split arguments ("-o dir") - .arg(format!("-o{}", base_dir.path().to_str().unwrap())) + .arg({ + let mut output_arg = OsString::from("-o"); + output_arg.push(base_dir.path()); + output_arg + }) .arg("--") .arg(patch_file) .stderr(Stdio::inherit()) .output() - .unwrap(); + .context(RunMailsplitSnafu)?; if !mailsplit.status.success() { - panic!("failed to apply patches"); + return MailsplitFailedSnafu { + status: mailsplit.status, + } + .fail(); } let mailsplit_patch_count = std::str::from_utf8(&mailsplit.stdout) - .unwrap() + .context(MailsplitOutputSnafu)? .trim() .parse() - .unwrap(); + .context(ParseMailsplitSnafu)?; Ok((1..=mailsplit_patch_count).map(move |patch_i| { base_dir.path().join( // Matches the format emitted by git-mailsplit @@ -64,56 +128,66 @@ impl Mailinfo { let mut subject = None; for patch_info_line in self.headers.lines() { if !patch_info_line.is_empty() { - match patch_info_line.split_once(": ").unwrap() { + match patch_info_line + .split_once(": ") + .context(MalformedMailHeaderSnafu)? + { ("Author", x) => author_name = Some(x), ("Email", x) => author_email = Some(x), ("Date", x) => date = Some(x), ("Subject", x) => subject = Some(x), - (header, _) => panic!("unknown header type {header:?}"), + (header, _) => return UnknownMailHeaderSnafu { header }.fail(), } } } - let date = OffsetDateTime::parse(date.unwrap(), &Rfc2822).unwrap(); + let date = date.context(NoDateSnafu)?; + let date = OffsetDateTime::parse(date, &Rfc2822).context(InvalidMailDateSnafu { date })?; + let subject = subject.context(NoSubjectSnafu)?.trim(); let full_msg = if self.rest_of_message.is_empty() { - subject.unwrap().trim().to_string() + subject.to_string() } else { - format!( - "{}\n\n{}", - subject.unwrap().trim(), - self.rest_of_message.trim() - ) + format!("{}\n\n{}", subject, self.rest_of_message.trim()) }; Ok(ParsedPatch { - subject: subject.unwrap().trim().to_string(), + subject: subject.to_string(), message: full_msg, author: Signature::new( - author_name.unwrap(), - author_email.unwrap(), + author_name.context(NoAuthorNameSnafu)?, + author_email.context(NoAuthorEmailSnafu)?, &git2::Time::new(date.unix_timestamp(), date.offset().whole_minutes().into()), ) - .unwrap(), - patch: Diff::from_buffer(&self.patch).unwrap(), + .context(InvalidSignatureSnafu)?, + patch: Diff::from_buffer(&self.patch).context(InvalidDiffSnafu)?, }) } } pub fn mailinfo(repo: &Repository, patch_email_file: &Path) -> Result { - let msg_file = NamedTempFile::new().unwrap().into_temp_path(); - let patch_file = NamedTempFile::new().unwrap().into_temp_path(); + let msg_file = NamedTempFile::new() + .context(CreateTempFileSnafu)? + .into_temp_path(); + let patch_file = NamedTempFile::new() + .context(CreateTempFileSnafu)? + .into_temp_path(); let mailinfo = raw_git_cmd(repo) .arg("mailinfo") .args([&msg_file, &patch_file]) - .stdin(File::open(patch_email_file).unwrap()) + .stdin(File::open(patch_email_file).context(OpenMailFileSnafu { + path: patch_email_file, + })?) .stderr(Stdio::inherit()) .output() - .unwrap(); + .context(RunMailinfoSnafu)?; if !mailinfo.status.success() { - panic!("failed to apply patches"); + return MailinfoFailedSnafu { + status: mailinfo.status, + } + .fail(); } - let patch_info = std::str::from_utf8(&mailinfo.stdout).unwrap(); + let patch_info = std::str::from_utf8(&mailinfo.stdout).context(MailinfoOutputSnafu)?; Ok(Mailinfo { headers: patch_info.to_string(), - rest_of_message: std::fs::read_to_string(msg_file).unwrap(), - patch: std::fs::read(patch_file).unwrap(), + rest_of_message: std::fs::read_to_string(msg_file).context(ReadMailinfoMessageSnafu)?, + patch: std::fs::read(patch_file).context(ReadMailinfoPatchSnafu)?, }) } From 92696d2b715391b73bc03c13e68474a152348297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 10 Feb 2025 12:41:16 +0100 Subject: [PATCH 25/48] Snafuize the remaining errors --- patchable/src/main.rs | 105 +++++++++++++++++++++++++++++++++--------- 1 file changed, 83 insertions(+), 22 deletions(-) diff --git a/patchable/src/main.rs b/patchable/src/main.rs index 5e43d4db1..1b11e5b3e 100644 --- a/patchable/src/main.rs +++ b/patchable/src/main.rs @@ -30,15 +30,16 @@ struct ProductVersionContext<'a> { } impl ProductVersionContext<'_> { - fn load_config(&self) -> ProductVersionConfig { + fn load_config(&self) -> Result { + let path = &self.config_path(); tracing::info!( - config.path = %self.config_path().display(), + config.path = ?path, "loading config" ); toml::from_str::( - &std::fs::read_to_string(self.config_path()).unwrap(), + &std::fs::read_to_string(path).context(LoadConfigSnafu { path })?, ) - .unwrap() + .context(ParseConfigSnafu { path }) } fn root(&self) -> PathBuf { @@ -90,6 +91,20 @@ enum Cmd { #[derive(Debug, Snafu)] pub enum Error { + #[snafu(display("failed to configure git logging"))] + ConfigureGitLogging { source: git2::Error }, + + #[snafu(display("failed to load config from {path:?}"))] + LoadConfig { + source: std::io::Error, + path: PathBuf, + }, + #[snafu(display("failed to parse config from {path:?}"))] + ParseConfig { + source: toml::de::Error, + path: PathBuf, + }, + #[snafu(display("failed to find images repository"))] FindImagesRepo { source: git2::Error }, #[snafu(display("images repository has no work directory"))] @@ -97,15 +112,47 @@ pub enum Error { #[snafu(display("failed to fetch patch series' base commit"))] FetchBaseCommit { source: repo::Error }, + #[snafu(display("failed to apply patch series"))] + ApplyPatches { source: patch::Error }, #[snafu(display("failed to open product repository"))] OpenProductRepoForCheckout { source: repo::Error }, #[snafu(display("failed to checkout product worktree"))] CheckoutProductWorktree { source: repo::Error }, + + #[snafu(display("failed to open product repository at {path:?}"))] + OpenProductRepo { source: git2::Error, path: PathBuf }, + #[snafu(display("failed to find base commit {commit:?} in repository {repo}"))] + FindBaseCommit { + source: git2::Error, + repo: error::RepoPath, + commit: String, + }, + #[snafu(display("failed to find head commit in repository {repo}"))] + FindHeadCommit { + source: git2::Error, + repo: error::RepoPath, + }, + #[snafu(display("failed to canonicalize history between {base}..{leaf}"))] + CanonicalizeHistory { + source: patch::Error, + base: error::CommitId, + leaf: error::CommitId, + }, + #[snafu(display( + "failed to format patches between {base}..{leaf} (canonicalized from {original_leaf})" + ))] + FormatPatches { + source: patch::Error, + base: error::CommitId, + leaf: error::CommitId, + original_leaf: error::CommitId, + }, } +type Result = std::result::Result; #[snafu::report] -fn main() -> Result<(), Error> { +fn main() -> Result<()> { tracing_subscriber::registry() .with(tracing_subscriber::fmt::layer()) .with( @@ -126,7 +173,7 @@ fn main() -> Result<(), Error> { git2::TraceLevel::Trace => tracing::trace!(target: "git", "{msg}"), } }) - .unwrap(); + .context(ConfigureGitLoggingSnafu)?; let opts = ::parse(); let images_repo = Repository::discover(".").context(FindImagesRepoSnafu)?; @@ -137,7 +184,7 @@ fn main() -> Result<(), Error> { pv, images_repo_root, }; - let config = ctx.load_config(); + let config = ctx.load_config()?; let product_repo_root = ctx.repo(); let product_repo = tracing::info_span!( "finding product repository", @@ -149,8 +196,8 @@ fn main() -> Result<(), Error> { let base_commit = repo::ensure_commit_exists_or_fetch(&product_repo, &config.base, &config.upstream) .context(FetchBaseCommitSnafu)?; - let patched_commit = - patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit).unwrap(); + let patched_commit = patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit) + .context(ApplyPatchesSnafu)?; let product_worktree_root = ctx.worktree_root(); repo::ensure_worktree_is_at( @@ -172,32 +219,42 @@ fn main() -> Result<(), Error> { pv, images_repo_root, }; - let config = ctx.load_config(); + let config = ctx.load_config()?; let product_worktree_root = ctx.worktree_root(); tracing::info!( worktree.root = ?product_worktree_root, "opening product worktree" ); - let product_version_repo = Repository::open(&product_worktree_root).unwrap(); + let product_version_repo = + Repository::open(&product_worktree_root).context(OpenProductRepoSnafu { + path: product_worktree_root, + })?; let base_commit = product_version_repo .revparse_single(&config.base) - .unwrap() - .peel_to_commit() - .unwrap() + .and_then(|c| c.peel_to_commit()) + .context(FindBaseCommitSnafu { + repo: &product_version_repo, + commit: config.base, + })? + .id(); + let original_leaf_commit = product_version_repo + .head() + .and_then(|c| c.peel_to_commit()) + .context(FindHeadCommitSnafu { + repo: &product_version_repo, + })? .id(); let canonical_leaf_commit = patch::canonicalize_commit_history( &product_version_repo, base_commit, - product_version_repo - .head() - .unwrap() - .peel_to_commit() - .unwrap() - .id(), + original_leaf_commit, ) - .unwrap(); + .context(CanonicalizeHistorySnafu { + base: base_commit, + leaf: original_leaf_commit, + })?; let patch_dir = ctx.patch_dir(); patch::format_patches( @@ -206,7 +263,11 @@ fn main() -> Result<(), Error> { base_commit, canonical_leaf_commit, ) - .unwrap(); + .context(FormatPatchesSnafu { + base: base_commit, + leaf: canonical_leaf_commit, + original_leaf: original_leaf_commit, + })?; tracing::info!( patch.dir = ?patch_dir, From 58f3ef4f665d7a2ad4aeeba74a878e7dc27a6b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 10 Feb 2025 12:48:02 +0100 Subject: [PATCH 26/48] Shrink the error types a bit --- patchable/src/error.rs | 4 ++-- patchable/src/patch_mail.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/patchable/src/error.rs b/patchable/src/error.rs index d0df4fa69..16084caf5 100644 --- a/patchable/src/error.rs +++ b/patchable/src/error.rs @@ -8,7 +8,7 @@ use std::{ use git2::{Commit, Object, Oid, Reference, Repository}; #[derive(Debug)] -pub struct CommitId(Oid); +pub struct CommitId(Box); impl Display for CommitId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.0) @@ -16,7 +16,7 @@ impl Display for CommitId { } impl From for CommitId { fn from(value: Oid) -> Self { - Self(value) + Self(Box::new(value)) } } impl From<&Commit<'_>> for CommitId { diff --git a/patchable/src/patch_mail.rs b/patchable/src/patch_mail.rs index 9d120ffae..6ff96d8a4 100644 --- a/patchable/src/patch_mail.rs +++ b/patchable/src/patch_mail.rs @@ -62,7 +62,8 @@ pub enum Error { NoSubject, #[snafu(display("failed to parse \"Date\" header (should be RFC2822)"))] InvalidMailDate { - source: time::error::Parse, + #[snafu(source(from(time::error::Parse, Box::new)))] + source: Box, date: String, }, #[snafu(display("failed to build commit signature from headers"))] From 25bda8dbfd8df5c252538278e195e0fc2d174cb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 10 Feb 2025 13:03:37 +0100 Subject: [PATCH 27/48] Move patchable to rust/patchable for consistency --- Cargo.toml | 2 +- {patchable => rust/patchable}/Cargo.toml | 0 {patchable => rust/patchable}/src/error.rs | 0 {patchable => rust/patchable}/src/main.rs | 0 {patchable => rust/patchable}/src/patch.rs | 0 {patchable => rust/patchable}/src/patch_mail.rs | 0 {patchable => rust/patchable}/src/repo.rs | 0 {patchable => rust/patchable}/src/utils.rs | 0 8 files changed, 1 insertion(+), 1 deletion(-) rename {patchable => rust/patchable}/Cargo.toml (100%) rename {patchable => rust/patchable}/src/error.rs (100%) rename {patchable => rust/patchable}/src/main.rs (100%) rename {patchable => rust/patchable}/src/patch.rs (100%) rename {patchable => rust/patchable}/src/patch_mail.rs (100%) rename {patchable => rust/patchable}/src/repo.rs (100%) rename {patchable => rust/patchable}/src/utils.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 47aa07586..a019301e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["patchable"] +members = ["rust/*"] resolver = "2" [workspace.dependencies] diff --git a/patchable/Cargo.toml b/rust/patchable/Cargo.toml similarity index 100% rename from patchable/Cargo.toml rename to rust/patchable/Cargo.toml diff --git a/patchable/src/error.rs b/rust/patchable/src/error.rs similarity index 100% rename from patchable/src/error.rs rename to rust/patchable/src/error.rs diff --git a/patchable/src/main.rs b/rust/patchable/src/main.rs similarity index 100% rename from patchable/src/main.rs rename to rust/patchable/src/main.rs diff --git a/patchable/src/patch.rs b/rust/patchable/src/patch.rs similarity index 100% rename from patchable/src/patch.rs rename to rust/patchable/src/patch.rs diff --git a/patchable/src/patch_mail.rs b/rust/patchable/src/patch_mail.rs similarity index 100% rename from patchable/src/patch_mail.rs rename to rust/patchable/src/patch_mail.rs diff --git a/patchable/src/repo.rs b/rust/patchable/src/repo.rs similarity index 100% rename from patchable/src/repo.rs rename to rust/patchable/src/repo.rs diff --git a/patchable/src/utils.rs b/rust/patchable/src/utils.rs similarity index 100% rename from patchable/src/utils.rs rename to rust/patchable/src/utils.rs From 7908127f786f2903ce30b522bd20c1f077a20170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 10 Feb 2025 14:32:30 +0100 Subject: [PATCH 28/48] Docs --- README.md | 18 ++++++++++++++++++ rust/patchable/README.md | 39 ++++++++++++++++++++++++++++++++++++++ rust/patchable/src/main.rs | 14 ++++++++++++++ rust/patchable/src/repo.rs | 7 +++++-- 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 rust/patchable/README.md diff --git a/README.md b/README.md index b56837435..d3da0fa7b 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,24 @@ bake --image-version 0.0.0-dev The GitHub action called `Build (and optionally publish) 0.0.0-dev images` can be triggered manually to do build all images in all versions. When triggered manually it will _not_ push the images to the registry. +## Patches + +Many products apply Stackable-specific patches, managed by [Patchable](rust/patchable). + +### Check out patched sources locally + +This is not required for building the images, but is useful when debugging or hacking on our patch sets. + +```sh +cargo patchable checkout druid 26.0.0 +``` + +### Save patched sources + +```sh +cargo patchable export druid 26.0.0 +``` + ## Verify Product Images To verify if Apache Zookeeper validate against OpenShift preflight, run: diff --git a/rust/patchable/README.md b/rust/patchable/README.md new file mode 100644 index 000000000..affc7ddba --- /dev/null +++ b/rust/patchable/README.md @@ -0,0 +1,39 @@ +# Patchable + +Patchable is a tool for managing patches for the third-party products distributed by Stackable as part of the Stackable Data Platform. + +Patchable works by keeping a series of .patch files (in `docker-images//stackable/patches/`) +as its source of truth, but using temporary Git repositories as an interface to modify those patches. +This lets us track the history of patches over time, but reuse the existing familiarity and tooling around Git. + +Patchable designates a commit as the upstream _base_ for each version, and considers each commit made on top of that +to be an individual patch. + +## Usage + +```sh +cargo patchable checkout druid 26.0.0 +pushd $(git rev-parse --show-toplevel)/druid/patchable-work/worktree/26.0.0/ +# do stuff +git commit +popd +cargo patchable export druid 26.0.0 +git status +``` + +For more details, run `cargo patchable --help`. + +## Notes + +- patchable only supports linear patch series (no merges beyond the base commit) +- patchable doesn't support support merging "materialized" trees, merge the .patch files instead, and `checkout`/`export` to update the hashes +- `patchable checkout` doesn't support resolving patch conflicts, use `git am` instead (and then `patchable export` the resolved patches) +- Always run patchable via `cargo patchable` (rather than `cargo install`ing it), to ensure that you use the correct version for a given checkout of docker-images + +## Configuration + +Patchable stores a per-version file in `docker-images//stackable/patches//patchable.toml`. +It currently recognizes the following keys: + +- `upstream` - the URL of the upstream repository (such as `https://github.com/apache/druid.git`) +- `base` - the commit hash of the upstream base commit (such as `7cffb81a8e124d5f218f9af16ad685acf5e9c67c`) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 1b11e5b3e..ed386b9e8 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -71,7 +71,12 @@ impl ProductVersionContext<'_> { } } +/// Patchable is a tool for managing patches for the third-party products distributed by Stackable. #[derive(clap::Parser)] +#[clap( + // Encourage people to let cargo decide the current version + bin_name = "cargo patchable", +)] struct Opts { #[clap(subcommand)] cmd: Cmd, @@ -79,10 +84,19 @@ struct Opts { #[derive(clap::Parser)] enum Cmd { + /// Check out a patched source tree to docker-images//patchable-work/worktree/ + /// + /// The patches will be pulled from docker-images//stackable/patches/ + /// + /// The source tree will be overwritten if it already exists (equivalent to `git switch`). Checkout { #[clap(flatten)] pv: ProductVersion, }, + + /// Export the patches from the source tree at docker-images//patchable-work/worktree/ + /// + /// The patches will be saved to docker-images//stackable/patches/ Export { #[clap(flatten)] pv: ProductVersion, diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index fe3cfd7b4..dfdf399f1 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -21,12 +21,15 @@ pub enum Error { branch: String, commit: error::CommitId, }, - #[snafu(display("failed to create worktree folder at {path:?}"))] + #[snafu(display("failed to create worktree parent folder at {path:?}"))] CreateWorktreePath { source: std::io::Error, path: PathBuf, }, - #[snafu(display("failed to create worktree {path:?} pointing at {branch} in {repo}"))] + #[snafu(display( + "failed to create worktree {path:?} pointing at {branch} in {repo} (hint: {})", + "it may have been created but deleted in the past, try `git -C {repo} worktree prune`" + ))] CreateWorktree { source: git2::Error, repo: error::RepoPath, From f21136d4899955c76e87f7ed43981aac6679dc4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 10 Feb 2025 14:51:19 +0100 Subject: [PATCH 29/48] Remove obsolete patchable.nu --- patchable.nu | 137 --------------------------------------------------- 1 file changed, 137 deletions(-) delete mode 100755 patchable.nu diff --git a/patchable.nu b/patchable.nu deleted file mode 100755 index 41d860ab5..000000000 --- a/patchable.nu +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env nu - -use std log - -def repo-root [] { - git rev-parse --show-toplevel -} - -def product-root [product: string] { - $"(repo-root)/($product)" -} - -def product-work-root [product: string] { - $"(product-root $product)/patchable-work" -} - -def product-repo [product: string, --upstream: string] { - let repo_path = $"(product-work-root $product)/product-repo" - log info $"Repository root for ($product) is ($repo_path)" - if not ($repo_path | path exists) { - log info $"Repository root not found, cloning from upstream ($upstream)" - git clone --bare $upstream $repo_path - } else { - log info "Repository root found, reusing" - } - $repo_path -} - -def product-version-worktree-root [product: string, version: string] { - $"(product-work-root $product)/worktree/($version)" -} - -def product-version-worktree-branch [version: string] { - $"patchable/($version)" -} - -def product-version-patch-dir [product: string, version: string] { - $"(product-root $product)/stackable/patches/($version)" -} - -def product-version-config [product: string, version: string] { - let path = $"(product-version-patch-dir $product $version)/patchable.toml" - log info $"Loading patch config from ($path)" - open $path -} - -def "main" [] { - print "Subcommands:" - print " ./patchable.nu checkout --help - Check out a product version from its patches" - print " ./patchable.nu export --help - Update a product's patches from its worktree" - print "" - print "Usage:" - print " $ ./patchable.nu checkout druid 26.0.0" - print " $ enter druid/patchable-work/worktree/26.0.0/" - print " $ # do stuff" - print " $ git commit" - print " $ dexit" - print " $ ./patchable.nu export druid 26.0.0" - print " $ git status" -} - -# Check out a patched source tree from its upstream sources with patches applied. -# -# If the source tree already exists it will be overwritten. Old commits can be recovered from the git reflog. -# -# A separate source tree is maintained for each product. -def "main checkout" [ - product: string # The name of the product (example: druid) - version: string # The version of the product (example: 26.0.0) - --force # Overwrite existing checkouts somewhat more aggressively -] { - let config = product-version-config $product $version - let product_repo = (product-repo $product --upstream=$config.upstream) - let worktree_path = product-version-worktree-root $product $version - let worktree_branch = product-version-worktree-branch $version - let $patch_dir = product-version-patch-dir $product $version - log info $"Worktree root is ($worktree_path)" - log info $"Worktree branch is ($worktree_branch), from base ($config.base) and patches at ($patch_dir)" - # These environment variables make git operate on the product worktree from now on - # $GIT_DIR must be the worktree's .git dir, even if that is just an alias for the backing repo, since each worktree maintains its own index - $env.GIT_DIR = $"($worktree_path)/.git" - $env.GIT_WORK_TREE = $worktree_path - if ($worktree_path | path exists) { - let worktree_git_dir = git rev-parse --git-dir - let worktree_rebase_progress_dir = $"($worktree_git_dir)/rebase-apply" - log info "Worktree root already exists, resetting" - if ($worktree_rebase_progress_dir | path exists) { - if $force { - log warning "Rebase/apply is in progress, aborting it" - git am --abort - } else { - error make {msg: "Rebase/apply is in progress, abort manually or pass --force flag"} - } - } - git checkout $config.base - } else { - log info "Worktree root not found, creating" - # $GIT_DIR won't exist yet, so we need to override it - git --git-dir $product_repo --work-tree $product_repo worktree add $worktree_path $config.base --detach - } - log info $"Creating work branch ($worktree_branch)" - git checkout -B $worktree_branch - log info "Importing patches" - let series_file = $"($patch_dir)/series" - if ($series_file | path exists) { - log info $"Series file exists at ($series_file), treating as stgit series" - git am $"($patch_dir)/series" --patch-format stgit-series - } else { - log info $"No series file found at ($series_file), treating as git mailbox" - git am ...(glob $"($patch_dir)/*.patch" | sort) - } -} - -# Export the patches in the current source tree of a product. -def "main export" [ - product: string # The name of the product (example: druid) - version: string # The version of the product (example: 26.0.0) -] { - let config = product-version-config $product $version - let product_repo = (product-repo $product --upstream=$config.upstream) - let worktree_path = product-version-worktree-root $product $version - let worktree_branch = product-version-worktree-branch $version - let $patch_dir = product-version-patch-dir $product $version - log info $"Worktree root is ($worktree_path)" - log info $"Worktree branch is ($worktree_branch), from base ($config.base) and patches at ($patch_dir)" - # These environment variables make git operate on the product worktree from now on - # $GIT_DIR must be the worktree's .git dir, even if that is just an alias for the backing repo, since each worktree maintains its own index - $env.GIT_DIR = $"($worktree_path)/.git" - $env.GIT_WORK_TREE = $worktree_path - log info "Deleting existing patches" - rm ...(glob $"($patch_dir)/{*.patch,series}" | tee { print $in }) - log info $"Exporting patches to ($patch_dir)" - git format-patch $config.base -o $patch_dir --base $config.base --keep-subject - # Normally the patches include their own commit IDs, which will change for every for every reimport - log info "Scrubbing commit ID from patches" - sed -i "1s/From [0-9a-f]\\+ /From 0000000000000000000000000000000000000000 /" ...(glob $"($patch_dir)/*.patch") -} From 3baa3e82ea81aa1101cd8712a9a8e3f97b143b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 10 Feb 2025 16:54:07 +0100 Subject: [PATCH 30/48] Add patchable init --- rust/patchable/README.md | 8 ++++ rust/patchable/src/main.rs | 80 ++++++++++++++++++++++++++++++++++++-- rust/patchable/src/repo.rs | 25 +++++++----- 3 files changed, 100 insertions(+), 13 deletions(-) diff --git a/rust/patchable/README.md b/rust/patchable/README.md index affc7ddba..3f198b9ff 100644 --- a/rust/patchable/README.md +++ b/rust/patchable/README.md @@ -37,3 +37,11 @@ It currently recognizes the following keys: - `upstream` - the URL of the upstream repository (such as `https://github.com/apache/druid.git`) - `base` - the commit hash of the upstream base commit (such as `7cffb81a8e124d5f218f9af16ad685acf5e9c67c`) + +### Template + +Instead of creating this manually, run `patchable init`: + +```toml +cargo patchable init druid 28.0.0 --upstream=https://github.com/apache/druid.git --base=druid-28.0.0 +``` diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index ed386b9e8..1921bf4a3 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -5,20 +5,29 @@ mod repo; mod utils; use core::str; -use std::path::{Path, PathBuf}; +use std::{ + fs::File, + io::Write, + path::{Path, PathBuf}, +}; use git2::Repository; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt as _, Snafu}; use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; #[derive(clap::Parser)] struct ProductVersion { + /// The product name slug (such as druid) product: String, + + /// The product version (such as 28.0.0) + /// + /// Should not contain a v prefix. version: String, } -#[derive(Deserialize)] +#[derive(Deserialize, Serialize)] struct ProductVersionConfig { upstream: String, base: String, @@ -101,6 +110,22 @@ enum Cmd { #[clap(flatten)] pv: ProductVersion, }, + + /// Creates a patchable.toml for a given product version + Init { + #[clap(flatten)] + pv: ProductVersion, + + /// The upstream URL (such as https://github.com/apache/druid.git) + #[clap(long)] + upstream: String, + + /// The upstream commit-ish (such as druid-28.0.0) that the patch series applies to + /// + /// Refs (such as tags and branches) will be resolved to commit IDs. + #[clap(long)] + base: String, + }, } #[derive(Debug, Snafu)] @@ -118,6 +143,18 @@ pub enum Error { source: toml::de::Error, path: PathBuf, }, + #[snafu(display("failed to serialize config"))] + SerializeConfig { source: toml::ser::Error }, + #[snafu(display("failed to create patch dir at {path:?}"))] + CreatePatchDir { + source: std::io::Error, + path: PathBuf, + }, + #[snafu(display("failed to save config to {path:?}"))] + SaveConfig { + source: std::io::Error, + path: PathBuf, + }, #[snafu(display("failed to find images repository"))] FindImagesRepo { source: git2::Error }, @@ -208,7 +245,7 @@ fn main() -> Result<()> { .context(OpenProductRepoForCheckoutSnafu)?; let base_commit = - repo::ensure_commit_exists_or_fetch(&product_repo, &config.base, &config.upstream) + repo::resolve_commitish_or_fetch(&product_repo, &config.base, &config.upstream) .context(FetchBaseCommitSnafu)?; let patched_commit = patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit) .context(ApplyPatchesSnafu)?; @@ -228,6 +265,7 @@ fn main() -> Result<()> { "worktree is ready!" ); } + Cmd::Export { pv } => { let ctx = ProductVersionContext { pv, @@ -288,6 +326,40 @@ fn main() -> Result<()> { "worktree is exported!" ); } + + Cmd::Init { pv, upstream, base } => { + let ctx = ProductVersionContext { + pv, + images_repo_root, + }; + + let product_repo_root = ctx.repo(); + let product_repo = tracing::info_span!( + "finding product repository", + product.repository = ?product_repo_root, + ) + .in_scope(|| repo::ensure_bare_repo(&product_repo_root)) + .context(OpenProductRepoForCheckoutSnafu)?; + + tracing::info!(?base, "resolving base commit-ish"); + let base_commit = repo::resolve_commitish_or_fetch(&product_repo, &base, &upstream) + .context(FetchBaseCommitSnafu)?; + tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); + + let config = ProductVersionConfig { + upstream, + base: base.to_string(), + }; + let config_path = ctx.config_path(); + if let Some(config_dir) = config_path.parent() { + std::fs::create_dir_all(config_dir) + .context(CreatePatchDirSnafu { path: config_dir })?; + } + let config_toml = toml::to_string_pretty(&config).context(SerializeConfigSnafu)?; + File::create_new(&config_path) + .and_then(|mut f| f.write_all(config_toml.as_bytes())) + .context(SaveConfigSnafu { path: config_path })?; + } } Ok(()) diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index dfdf399f1..1a4e91c76 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -57,7 +57,7 @@ pub enum Error { target: error::CommitRef, }, - #[snafu(display("failed to find {commit} in {repo}"))] + #[snafu(display("failed to find commit {commit} in {repo}"))] FindCommit { source: git2::Error, repo: error::RepoPath, @@ -105,14 +105,16 @@ pub fn ensure_bare_repo(path: &Path) -> Result { } } -/// Fetch `commit` from `upstream_url`, if it doesn't already exist. +/// Try to resolve `commitish` locally. If it doesn't exist, try to fetch it from `upstream_url`. +/// +/// Returns the resolved commit ID. #[tracing::instrument(skip(repo))] -pub fn ensure_commit_exists_or_fetch( +pub fn resolve_commitish_or_fetch( repo: &Repository, - commit: &str, + commitish: &str, upstream_url: &str, ) -> Result { - let commit = match repo.revparse_single(commit) { + let commit = match repo.revparse_single(commitish) { Ok(commit_obj) => { tracing::info!("base commit exists, reusing"); Ok(commit_obj) @@ -128,9 +130,10 @@ pub fn ensure_commit_exists_or_fetch( url: upstream_url, })? .fetch( - &[commit], + &[commitish], Some( FetchOptions::new() + .update_fetchhead(true) // TODO: could be 1, CLI option maybe? .depth(0), ), @@ -139,14 +142,18 @@ pub fn ensure_commit_exists_or_fetch( .with_context(|_| FetchSnafu { repo, url: upstream_url, - refs: vec![commit.to_string()], + refs: vec![commitish.to_string()], })?; tracing::info!("fetched base commit"); - repo.revparse_single(commit) + // FETCH_HEAD is written by Remote::fetch to be the last reference fetched + repo.revparse_single("FETCH_HEAD") } Err(err) => Err(err), } - .context(FindCommitSnafu { repo, commit })?; + .context(FindCommitSnafu { + repo, + commit: commitish, + })?; Ok(commit .peel_to_commit() .context(FindCommitSnafu { repo, commit })? From e5b8282514b257d73937cede3c00cfe277b90037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 10 Feb 2025 17:21:03 +0100 Subject: [PATCH 31/48] Enforce that base must be a commit ID in patchable.toml --- rust/patchable/src/main.rs | 37 ++++++++++++++++++++++--------------- rust/patchable/src/utils.rs | 16 ++++++++++++++++ 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 1921bf4a3..2a7fc0a42 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -11,7 +11,7 @@ use std::{ path::{Path, PathBuf}, }; -use git2::Repository; +use git2::{Oid, Repository}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt as _, Snafu}; use tracing_subscriber::{layer::SubscriberExt as _, util::SubscriberInitExt as _}; @@ -30,7 +30,8 @@ struct ProductVersion { #[derive(Deserialize, Serialize)] struct ProductVersionConfig { upstream: String, - base: String, + #[serde(with = "utils::oid_serde")] + base: Oid, } struct ProductVersionContext<'a> { @@ -244,9 +245,12 @@ fn main() -> Result<()> { .in_scope(|| repo::ensure_bare_repo(&product_repo_root)) .context(OpenProductRepoForCheckoutSnafu)?; - let base_commit = - repo::resolve_commitish_or_fetch(&product_repo, &config.base, &config.upstream) - .context(FetchBaseCommitSnafu)?; + let base_commit = repo::resolve_commitish_or_fetch( + &product_repo, + &config.base.to_string(), + &config.upstream, + ) + .context(FetchBaseCommitSnafu)?; let patched_commit = patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit) .context(ApplyPatchesSnafu)?; @@ -283,14 +287,7 @@ fn main() -> Result<()> { path: product_worktree_root, })?; - let base_commit = product_version_repo - .revparse_single(&config.base) - .and_then(|c| c.peel_to_commit()) - .context(FindBaseCommitSnafu { - repo: &product_version_repo, - commit: config.base, - })? - .id(); + let base_commit = config.base; let original_leaf_commit = product_version_repo .head() .and_then(|c| c.peel_to_commit()) @@ -341,14 +338,17 @@ fn main() -> Result<()> { .in_scope(|| repo::ensure_bare_repo(&product_repo_root)) .context(OpenProductRepoForCheckoutSnafu)?; + // --base can be a reference, but patchable.toml should always have a resolved commit id, + // so that it cannot be changed under our feet (without us knowing so, anyway...). tracing::info!(?base, "resolving base commit-ish"); let base_commit = repo::resolve_commitish_or_fetch(&product_repo, &base, &upstream) .context(FetchBaseCommitSnafu)?; tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); + tracing::info!("saving configuration"); let config = ProductVersionConfig { upstream, - base: base.to_string(), + base: base_commit, }; let config_path = ctx.config_path(); if let Some(config_dir) = config_path.parent() { @@ -358,7 +358,14 @@ fn main() -> Result<()> { let config_toml = toml::to_string_pretty(&config).context(SerializeConfigSnafu)?; File::create_new(&config_path) .and_then(|mut f| f.write_all(config_toml.as_bytes())) - .context(SaveConfigSnafu { path: config_path })?; + .context(SaveConfigSnafu { path: &config_path })?; + + tracing::info!( + config.path = ?config_path, + product = ctx.pv.product, + version = ctx.pv.version, + "created configuration for product version" + ); } } diff --git a/rust/patchable/src/utils.rs b/rust/patchable/src/utils.rs index 7a165484f..f71557a28 100644 --- a/rust/patchable/src/utils.rs +++ b/rust/patchable/src/utils.rs @@ -14,3 +14,19 @@ pub fn raw_git_cmd(repo: &Repository) -> std::process::Command { ); cmd } + +/// Implements (equivalents of) the [`serde`] traits over [`git2::Oid`]. +/// +/// For use with `#[serde(with = ...)]`. +pub mod oid_serde { + use git2::Oid; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + pub fn serialize(value: &Oid, ser: S) -> Result { + value.to_string().serialize(ser) + } + pub fn deserialize<'de, D: Deserializer<'de>>(de: D) -> Result { + String::deserialize(de) + .and_then(|oid| Oid::from_str(&oid).map_err(::custom)) + } +} From 4dee9ec67093dca43be412c9d9b92b7914e488cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 10 Feb 2025 17:41:26 +0100 Subject: [PATCH 32/48] Ensure that init always fetches base from upstream --- rust/patchable/src/main.rs | 10 ++-------- rust/patchable/src/repo.rs | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 2a7fc0a42..7796a396b 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -174,12 +174,6 @@ pub enum Error { #[snafu(display("failed to open product repository at {path:?}"))] OpenProductRepo { source: git2::Error, path: PathBuf }, - #[snafu(display("failed to find base commit {commit:?} in repository {repo}"))] - FindBaseCommit { - source: git2::Error, - repo: error::RepoPath, - commit: String, - }, #[snafu(display("failed to find head commit in repository {repo}"))] FindHeadCommit { source: git2::Error, @@ -245,7 +239,7 @@ fn main() -> Result<()> { .in_scope(|| repo::ensure_bare_repo(&product_repo_root)) .context(OpenProductRepoForCheckoutSnafu)?; - let base_commit = repo::resolve_commitish_or_fetch( + let base_commit = repo::resolve_and_fetch_commitish( &product_repo, &config.base.to_string(), &config.upstream, @@ -341,7 +335,7 @@ fn main() -> Result<()> { // --base can be a reference, but patchable.toml should always have a resolved commit id, // so that it cannot be changed under our feet (without us knowing so, anyway...). tracing::info!(?base, "resolving base commit-ish"); - let base_commit = repo::resolve_commitish_or_fetch(&product_repo, &base, &upstream) + let base_commit = repo::resolve_and_fetch_commitish(&product_repo, &base, &upstream) .context(FetchBaseCommitSnafu)?; tracing::info!(?base, base.commit = ?base_commit, "resolved base commit"); diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 1a4e91c76..07f926ae2 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -105,24 +105,29 @@ pub fn ensure_bare_repo(path: &Path) -> Result { } } -/// Try to resolve `commitish` locally. If it doesn't exist, try to fetch it from `upstream_url`. +/// Try to resolve and fetch `commitish` from `upstream_url`. +/// +/// As an optimization, it can skip fetching if `commitish` is a literal commit ID that exists locally. /// /// Returns the resolved commit ID. #[tracing::instrument(skip(repo))] -pub fn resolve_commitish_or_fetch( +pub fn resolve_and_fetch_commitish( repo: &Repository, commitish: &str, upstream_url: &str, ) -> Result { - let commit = match repo.revparse_single(commitish) { + let oid = Oid::from_str(commitish); + let commitish_is_oid = oid.is_ok(); + let local_commit = oid.and_then(|oid| repo.find_commit(oid)); + let commit = match local_commit { Ok(commit_obj) => { - tracing::info!("base commit exists, reusing"); + tracing::info!("literal commit exists locally, reusing"); Ok(commit_obj) } - Err(err) if err.code() == git2::ErrorCode::NotFound => { + Err(err) if !commitish_is_oid || err.code() == git2::ErrorCode::NotFound => { tracing::info!( error = &err as &dyn std::error::Error, - "base commit not found, fetching from upstream" + "base commit not found locally, fetching from upstream" ); repo.remote_anonymous(upstream_url) .context(CreateRemoteSnafu { @@ -147,6 +152,7 @@ pub fn resolve_commitish_or_fetch( tracing::info!("fetched base commit"); // FETCH_HEAD is written by Remote::fetch to be the last reference fetched repo.revparse_single("FETCH_HEAD") + .and_then(|obj| obj.peel_to_commit()) } Err(err) => Err(err), } @@ -154,10 +160,7 @@ pub fn resolve_commitish_or_fetch( repo, commit: commitish, })?; - Ok(commit - .peel_to_commit() - .context(FindCommitSnafu { repo, commit })? - .id()) + Ok(commit.id()) } /// Ensure that the worktree at `worktree_root` exists and is checked out at `branch`. From 2a75edd1b30e7cdd795cd096e4e3e70cba0011c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 13 Feb 2025 16:55:44 +0100 Subject: [PATCH 33/48] Print worktree directory on checkout Also move logs to stderr to avoid confusion --- rust/patchable/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 7796a396b..d963d6e23 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -200,7 +200,7 @@ type Result = std::result::Result; #[snafu::report] fn main() -> Result<()> { tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer()) + .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) .with( tracing_subscriber::EnvFilter::builder() .with_default_directive(tracing_subscriber::filter::LevelFilter::INFO.into()) @@ -262,6 +262,9 @@ fn main() -> Result<()> { worktree.root = ?product_worktree_root, "worktree is ready!" ); + + // Print directory so you can run `cd $(cargo patchable checkout ...)` + println!("{}", product_worktree_root.display()); } Cmd::Export { pv } => { From ffae725bc01efd9bc074e867efef526a54ab6fed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 13 Feb 2025 17:32:37 +0100 Subject: [PATCH 34/48] Document how to rebase patch series --- README.md | 29 ++++++++++++++++++++++++++++- rust/patchable/README.md | 3 +-- rust/patchable/src/main.rs | 32 +++++++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d3da0fa7b..98a7432aa 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Many products apply Stackable-specific patches, managed by [Patchable](rust/patc This is not required for building the images, but is useful when debugging or hacking on our patch sets. ```sh -cargo patchable checkout druid 26.0.0 +cd $(cargo patchable checkout druid 26.0.0) ``` ### Save patched sources @@ -73,6 +73,33 @@ cargo patchable checkout druid 26.0.0 cargo patchable export druid 26.0.0 ``` +### Porting patch series to a new version + +Patchable doesn't support restoring a patch series that doesn't apply cleanly. Instead, use `git cherry-pick` to rebase the patch series. + +For example, let's try rebasing our patch series from Druid 26.0.0 to Druid 28.0.0 (which is not packaged by SDP): + +```sh +# Restore the old version +# In addition to creating the version worktree, this also creates the branches patchable/26.0.0 (26.0.0 with our patches applied) and +# patchable/base/26.0.0 (upstream 26.0.0 with no patches). +cargo patchable checkout druid 26.0.0 +# Tell Patchable about the new version 28.0.0, which can be fetched from https://github.com/apache/druid.git, and has the tag druid-28.0.0 +cargo patchable init druid 28.0.0 --upstream https://github.com/apache/druid.git --base druid-28.0.0 +# Create and go to the worktree for the new version +pushd $(cargo patchable checkout druid 28.0.0) + +# Rebase the patch series +git cherry-pick patchable/base/26.0.0..patchable/26.0.0 +# Solve conflicts and `git cherry-pick --continue` until done +# You can also use `git cherry-pick --skip` to skip patches that are no longer required + +# Leave and export the new patch series! +popd +cargo patchable export druid 28.0.0 +git status +``` + ## Verify Product Images To verify if Apache Zookeeper validate against OpenShift preflight, run: diff --git a/rust/patchable/README.md b/rust/patchable/README.md index 3f198b9ff..a683413d3 100644 --- a/rust/patchable/README.md +++ b/rust/patchable/README.md @@ -12,8 +12,7 @@ to be an individual patch. ## Usage ```sh -cargo patchable checkout druid 26.0.0 -pushd $(git rev-parse --show-toplevel)/druid/patchable-work/worktree/26.0.0/ +pushd $(cargo patchable checkout druid 26.0.0) # do stuff git commit popd diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index d963d6e23..1f2a33058 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -76,6 +76,10 @@ impl ProductVersionContext<'_> { self.work_root().join("worktree").join(&self.pv.version) } + fn base_branch(&self) -> String { + format!("patchable/base/{}", self.pv.version) + } + fn worktree_branch(&self) -> String { format!("patchable/{}", self.pv.version) } @@ -245,21 +249,47 @@ fn main() -> Result<()> { &config.upstream, ) .context(FetchBaseCommitSnafu)?; + let base_branch = ctx.base_branch(); + let base_branch = match product_repo + .find_commit(base_commit) + .and_then(|base| product_repo.branch(&base_branch, &base, true)) + { + Ok(_) => { + tracing::info!( + branch.base = base_branch, + branch.base.commit = %base_commit, + "updated base branch" + ); + Some(base_branch) + } + Err(err) => { + tracing::warn!( + error = &err as &dyn std::error::Error, + branch.base = base_branch, + branch.base.commit = %base_commit, + "failed to update base branch reference, ignoring..." + ); + None + } + }; let patched_commit = patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit) .context(ApplyPatchesSnafu)?; let product_worktree_root = ctx.worktree_root(); + let worktree_branch = ctx.worktree_branch(); repo::ensure_worktree_is_at( &product_repo, &ctx.pv.version, &product_worktree_root, - &ctx.worktree_branch(), + &worktree_branch, patched_commit, ) .context(CheckoutProductWorktreeSnafu)?; tracing::info!( worktree.root = ?product_worktree_root, + branch.worktree = worktree_branch, + branch.base = base_branch, "worktree is ready!" ); From 59677957150a99fd189641d4a2ccda7118644d89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 13 Feb 2025 17:34:23 +0100 Subject: [PATCH 35/48] Remove dummy patches --- .../stackable/patches/26.0.0/0009-asdf.patch | 25 ------------------- .../stackable/patches/26.0.0/0010-qwer.patch | 25 ------------------- 2 files changed, 50 deletions(-) delete mode 100644 druid/stackable/patches/26.0.0/0009-asdf.patch delete mode 100644 druid/stackable/patches/26.0.0/0010-qwer.patch diff --git a/druid/stackable/patches/26.0.0/0009-asdf.patch b/druid/stackable/patches/26.0.0/0009-asdf.patch deleted file mode 100644 index 4bb0252d0..000000000 --- a/druid/stackable/patches/26.0.0/0009-asdf.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 929707d95a423bda607cf8980c17bd7182c7b808 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= -Date: Thu, 12 Dec 2024 18:26:17 +0100 -Subject: asdf - ---- - upload.sh | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/upload.sh b/upload.sh -index 2b49df1a49..3927c205e3 100755 ---- a/upload.sh -+++ b/upload.sh -@@ -18,6 +18,8 @@ - # Script to upload tarball of assembly build to static.druid.io for serving - # - -+echo a2 -+ - if [ $# -lt 1 ]; then - echo "Usage: $0 " >&2 - exit 2 --- -2.48.1 - diff --git a/druid/stackable/patches/26.0.0/0010-qwer.patch b/druid/stackable/patches/26.0.0/0010-qwer.patch deleted file mode 100644 index 95fbc9501..000000000 --- a/druid/stackable/patches/26.0.0/0010-qwer.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 8de53941c8ad36144b598f14cd17a140333c499d Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= -Date: Thu, 12 Dec 2024 18:27:00 +0100 -Subject: qwer - ---- - upload.sh | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/upload.sh b/upload.sh -index 3927c205e3..493739e181 100755 ---- a/upload.sh -+++ b/upload.sh -@@ -25,6 +25,8 @@ if [ $# -lt 1 ]; then - exit 2 - fi - -+echo b -+ - VERSION=$1 - DRUID_TAR=druid-$VERSION-bin.tar.gz - MYSQL_TAR=mysql-metadata-storage-$VERSION.tar.gz --- -2.48.1 - From 8a65c10893737dbafbc41af3b92ffa16e5b8bbbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 13 Feb 2025 17:56:18 +0100 Subject: [PATCH 36/48] Documentation and cleanup --- README.md | 6 ++++++ rust/patchable/src/main.rs | 32 ++++++++++++++++++++------------ rust/patchable/src/patch.rs | 5 +---- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 98a7432aa..c7055af3d 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,12 @@ cd $(cargo patchable checkout druid 26.0.0) cargo patchable export druid 26.0.0 ``` +### Initialize a new patch series + +```sh +cargo patchable init druid 28.0.0 --upstream https://github.com/apache/druid.git --base druid-28.0.0 +``` + ### Porting patch series to a new version Patchable doesn't support restoring a patch series that doesn't apply cleanly. Instead, use `git cherry-pick` to rebase the patch series. diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 1f2a33058..2c086efe6 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -52,34 +52,46 @@ impl ProductVersionContext<'_> { .context(ParseConfigSnafu { path }) } - fn root(&self) -> PathBuf { + /// The root directory for files related to the product (across all versions). + fn product_dir(&self) -> PathBuf { self.images_repo_root.join(&self.pv.product) } + /// The directory containing patches for the product version. fn patch_dir(&self) -> PathBuf { - self.root().join("stackable/patches").join(&self.pv.version) + self.product_dir() + .join("stackable/patches") + .join(&self.pv.version) } + /// The patchable configuration file for the product version. fn config_path(&self) -> PathBuf { self.patch_dir().join("patchable.toml") } + /// The directory containing all ephemeral data used by patchable for the product (across all versions). + /// + /// Should be gitignored, and can safely be deleted as long as all relevant versions have been `patchable export`ed. fn work_root(&self) -> PathBuf { - self.root().join("patchable-work") + self.product_dir().join("patchable-work") } - fn repo(&self) -> PathBuf { + /// The repository for the product (across all versions). + fn product_repo(&self) -> PathBuf { self.work_root().join("product-repo") } + /// The worktree root for the product version. fn worktree_root(&self) -> PathBuf { self.work_root().join("worktree").join(&self.pv.version) } + /// Branch pointing at the upstream base commit for the product version. fn base_branch(&self) -> String { format!("patchable/base/{}", self.pv.version) } + /// branch pointing at the last commit in the patch series for the product version. fn worktree_branch(&self) -> String { format!("patchable/{}", self.pv.version) } @@ -235,13 +247,9 @@ fn main() -> Result<()> { images_repo_root, }; let config = ctx.load_config()?; - let product_repo_root = ctx.repo(); - let product_repo = tracing::info_span!( - "finding product repository", - product.repository = ?product_repo_root, - ) - .in_scope(|| repo::ensure_bare_repo(&product_repo_root)) - .context(OpenProductRepoForCheckoutSnafu)?; + let product_repo_root = ctx.product_repo(); + let product_repo = repo::ensure_bare_repo(&product_repo_root) + .context(OpenProductRepoForCheckoutSnafu)?; let base_commit = repo::resolve_and_fetch_commitish( &product_repo, @@ -357,7 +365,7 @@ fn main() -> Result<()> { images_repo_root, }; - let product_repo_root = ctx.repo(); + let product_repo_root = ctx.product_repo(); let product_repo = tracing::info_span!( "finding product repository", product.repository = ?product_repo_root, diff --git a/rust/patchable/src/patch.rs b/rust/patchable/src/patch.rs index 5e685470e..60b9c0668 100644 --- a/rust/patchable/src/patch.rs +++ b/rust/patchable/src/patch.rs @@ -171,10 +171,7 @@ fn patch_files(patch_dir: &Path) -> Result> { /// even if those files are modified by some of the patches) #[tracing::instrument(skip(repo))] pub fn apply_patches(repo: &Repository, patch_dir: &Path, base_commit: Oid) -> Result { - tracing::info!( - patch.dir = %patch_dir.display(), - "applying patches" - ); + tracing::info!("applying patches"); let mut last_commit_id = base_commit; for ref patch_file in patch_files(patch_dir)? { tracing::info!( From 07d05debad6f1d5883bac29d002846d55c511ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 13 Feb 2025 18:00:54 +0100 Subject: [PATCH 37/48] Remove git version suffix from patches --- ...1-Removes-all-traces-of-the-druid-ranger-extension.patch | 3 --- .../0002-Include-Prometheus-emitter-in-distribution.patch | 3 --- .../26.0.0/0003-Stop-building-unused-extensions.patch | 3 --- ...dates-all-dependencies-that-have-a-new-patch-relea.patch | 3 --- .../0005-Include-jackson-dataformat-xml-dependency.patch | 3 --- .../26.0.0/0006-Stop-building-the-tar.gz-distribution.patch | 3 --- .../patches/26.0.0/0007-Update-CycloneDX-plugin.patch | 3 --- .../stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch | 3 --- rust/patchable/src/patch.rs | 6 +++++- 9 files changed, 5 insertions(+), 25 deletions(-) diff --git a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch index 3ab5b674f..6823e2c61 100644 --- a/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch +++ b/druid/stackable/patches/26.0.0/0001-Removes-all-traces-of-the-druid-ranger-extension.patch @@ -42,6 +42,3 @@ index 0c6294f5ed..a33c6bd521 100644 extensions-core/druid-catalog extensions-core/testing-tools --- -2.48.1 - diff --git a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch index 153955a4a..3bc040817 100644 --- a/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch +++ b/druid/stackable/patches/26.0.0/0002-Include-Prometheus-emitter-in-distribution.patch @@ -64,6 +64,3 @@ index a6e72cf2c2..3ab13d5d11 100644 integration-test --- -2.48.1 - diff --git a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch index c59586fac..722e9e42a 100644 --- a/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch +++ b/druid/stackable/patches/26.0.0/0003-Stop-building-unused-extensions.patch @@ -69,6 +69,3 @@ index a33c6bd521..f5001910e1 100644 ${repoOrgId} --- -2.48.1 - diff --git a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch index f3028c6d4..53c20d559 100644 --- a/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch +++ b/druid/stackable/patches/26.0.0/0004-Updates-all-dependencies-that-have-a-new-patch-relea.patch @@ -132,6 +132,3 @@ index f5001910e1..2364f27dc4 100644 3.5.10 2.5.7 --- -2.48.1 - diff --git a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch index e65434626..4032142ab 100644 --- a/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch +++ b/druid/stackable/patches/26.0.0/0005-Include-jackson-dataformat-xml-dependency.patch @@ -27,6 +27,3 @@ index fdc6f1f548..9f18e614e9 100644 com.fasterxml.jackson.datatype jackson-datatype-joda --- -2.48.1 - diff --git a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch index 91f7c3edd..910a7a0a5 100644 --- a/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch +++ b/druid/stackable/patches/26.0.0/0006-Stop-building-the-tar.gz-distribution.patch @@ -22,6 +22,3 @@ index ff8e0d2fdd..f9daa49e21 100644 --- -2.48.1 - diff --git a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch index eeed90fb8..36756ca94 100644 --- a/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch +++ b/druid/stackable/patches/26.0.0/0007-Update-CycloneDX-plugin.patch @@ -24,6 +24,3 @@ index 2364f27dc4..c902899304 100644 package --- -2.48.1 - diff --git a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch index c3628ffd3..7368f95e7 100644 --- a/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch +++ b/druid/stackable/patches/26.0.0/0008-Fix-CVE-2024-36114.patch @@ -42,6 +42,3 @@ index c902899304..6c24bdc0b2 100644 commons-codec --- -2.48.1 - diff --git a/rust/patchable/src/patch.rs b/rust/patchable/src/patch.rs index 60b9c0668..899fe6988 100644 --- a/rust/patchable/src/patch.rs +++ b/rust/patchable/src/patch.rs @@ -345,7 +345,11 @@ pub fn format_patches( .arg(format!("{base_commit}..{leaf_commit}")) .arg("-o") .arg(patch_dir) - .arg("--keep-subject") + .args([ + "--keep-subject", + // By default, git includes its own version number as a suffix, which makes patches unstable across git versions + "--no-signature", + ]) .status() .context(RunFormatMailSnafu)?; if !status.success() { From d63f73ddd2ae23a84e2fa0d9eaa65a53fa8dbc9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 13 Feb 2025 18:10:41 +0100 Subject: [PATCH 38/48] Skip comments in series file --- rust/patchable/src/patch.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/patchable/src/patch.rs b/rust/patchable/src/patch.rs index 899fe6988..b29504c17 100644 --- a/rust/patchable/src/patch.rs +++ b/rust/patchable/src/patch.rs @@ -128,6 +128,8 @@ fn patch_files(patch_dir: &Path) -> Result> { "series file found, treating as stgit series" ); file.lines() + // Skip comment lines + .filter(|line| !line.starts_with('#')) .map(|file_name| patch_dir.join(file_name)) .collect::>() } From 5dfb933d4492490c86f272482625f5558e8304ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Thu, 13 Feb 2025 19:40:21 +0100 Subject: [PATCH 39/48] Document how to import invalid patch series into patchable --- README.md | 21 +++++++++++++++++++++ rust/patchable/src/main.rs | 15 ++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c7055af3d..ce2767ff6 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,27 @@ cargo patchable export druid 26.0.0 cargo patchable init druid 28.0.0 --upstream https://github.com/apache/druid.git --base druid-28.0.0 ``` +### Importing patch series into Patchable + +Patchable is stricter about applying invalid patches (both metadata and patches themselves) than Git is. + +If an initial `cargo patchable checkout` fails then `git am` can be useful for the initial migration: + +```sh +# Create Patchable configuration for the new version, if it doesn't already exist +cargo patchable init druid 30.0.0 --upstream https://github.com/apache/druid.git --base druid-30.0.0 +# Check out the upstream base commit, without trying to apply the patches +pushd $(cargo patchable checkout druid 30.0.0 --base-only) + +# Apply the patch series +git am ../../../stackable/patches/30.0.0/*.patch +# Resolve any conflicts that arise, and `git am --continue` until done + +# Leave and export the new patch series! +popd +cargo patchable export druid 30.0.0 +``` + ### Porting patch series to a new version Patchable doesn't support restoring a patch series that doesn't apply cleanly. Instead, use `git cherry-pick` to rebase the patch series. diff --git a/rust/patchable/src/main.rs b/rust/patchable/src/main.rs index 2c086efe6..45c180842 100644 --- a/rust/patchable/src/main.rs +++ b/rust/patchable/src/main.rs @@ -118,6 +118,10 @@ enum Cmd { Checkout { #[clap(flatten)] pv: ProductVersion, + + /// Check out the base commit, without applying patches + #[clap(long)] + base_only: bool, }, /// Export the patches from the source tree at docker-images//patchable-work/worktree/ @@ -241,7 +245,7 @@ fn main() -> Result<()> { let images_repo = Repository::discover(".").context(FindImagesRepoSnafu)?; let images_repo_root = images_repo.workdir().context(NoImagesRepoWorkdirSnafu)?; match opts.cmd { - Cmd::Checkout { pv } => { + Cmd::Checkout { pv, base_only } => { let ctx = ProductVersionContext { pv, images_repo_root, @@ -280,8 +284,13 @@ fn main() -> Result<()> { None } }; - let patched_commit = patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit) - .context(ApplyPatchesSnafu)?; + let patched_commit = if !base_only { + patch::apply_patches(&product_repo, &ctx.patch_dir(), base_commit) + .context(ApplyPatchesSnafu)? + } else { + tracing::warn!("--base-only specified, skipping patches"); + base_commit + }; let product_worktree_root = ctx.worktree_root(); let worktree_branch = ctx.worktree_branch(); From 205d8312ffac2fffe2cb85b34ef6aa26067d769e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 14 Feb 2025 16:52:39 +0100 Subject: [PATCH 40/48] More documentation --- README.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ce2767ff6..c56b41e97 100644 --- a/README.md +++ b/README.md @@ -59,24 +59,52 @@ When triggered manually it will _not_ push the images to the registry. Many products apply Stackable-specific patches, managed by [Patchable](rust/patchable). +Patchable doesn't _edit_ anything by itself. Instead, it's a uniform way to apply a set of patches +to an upstream Git repository, and then export your local changes back into patch files. +You can then edit the branch created by patchable using any Git frontend, such as the git CLI or +[jj](https://jj-vcs.github.io/jj/latest/). + +This way, the patch files are the global source of truth and track the history of our patch series, +while you can still use the same familiar Git tools to manipulate them. + ### Check out patched sources locally -This is not required for building the images, but is useful when debugging or hacking on our patch sets. +> [!NOTE] +> This is not required for building images, but is used for when hacking on or debugging patch series. ```sh -cd $(cargo patchable checkout druid 26.0.0) -``` +# Fetches the upstream repository (if required), and creates a git worktree to work with it +# It also creates two branches: +# - patchable/{version} (HEAD, has all patches applied) +# - patchable/base/{version} (the upstream) +pushd $(cargo patchable checkout druid 26.0.0) -### Save patched sources +# Commit to add new patches +git commit -```sh +# Rebase against the base commit to edit or remove patches +git rebase --interactive patchable/base/26.0.0 +# jj edit also works, but make sure to go back to the tip before exporting + +# When done, export your patches and commit them (to docker-images) +popd cargo patchable export druid 26.0.0 +git status ``` +> ![IMPORTANT] +> `cargo patchable export` exports whatever is currently checked out (`HEAD`) in the worktree. +> If you use `jj edit` (or `git switch`) then you _must_ go back to the tip before exporting, or +> any patches after that point will be omitted from the export. + ### Initialize a new patch series +Patchable stores metadata about each patch series in its `patchable.toml`, and will not be able to check out +a patch series that lacks one. It can be generated using `cargo patchable init`: + ```sh cargo patchable init druid 28.0.0 --upstream https://github.com/apache/druid.git --base druid-28.0.0 +cargo patchable checkout druid 28.0.0 ``` ### Importing patch series into Patchable @@ -116,10 +144,13 @@ cargo patchable init druid 28.0.0 --upstream https://github.com/apache/druid.git # Create and go to the worktree for the new version pushd $(cargo patchable checkout druid 28.0.0) -# Rebase the patch series +# Cherry pick the old patch series git cherry-pick patchable/base/26.0.0..patchable/26.0.0 # Solve conflicts and `git cherry-pick --continue` until done -# You can also use `git cherry-pick --skip` to skip patches that are no longer required +# You can also use `git cherry-pick --skip` to skip resolving conflicts for patches that are no longer required + +# If some patches are no longer required, use an interactive rebase to remove them (or do other cleanup) +git rebase --interactive patchable/base/28.0.0 # Leave and export the new patch series! popd @@ -127,6 +158,17 @@ cargo patchable export druid 28.0.0 git status ``` +### Porting patches between versions + +Individual patches can also be cherry-picked across versions. + +For example, assuming we are in the Druid 28.0.0 workspace and want to port the last patch of the Druid 26.0.0 series: + +```sh +# git cherry-pick is also fine for grabbing arbitrary patches +git cherry-pick patchable/26.0.0 +``` + ## Verify Product Images To verify if Apache Zookeeper validate against OpenShift preflight, run: From 70203696d4b53f70cd3fac655752899f4a7b917e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 14 Feb 2025 17:00:57 +0100 Subject: [PATCH 41/48] Add openssl to shell.nix --- shell.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/shell.nix b/shell.nix index 09a26e528..1782dff73 100644 --- a/shell.nix +++ b/shell.nix @@ -7,7 +7,15 @@ let bake = pkgs.callPackage (sources.image-tools + "/image-tools.nix") { }; in pkgs.mkShell { - packages = with pkgs; [ + packages = [ bake ]; + + buildInputs = [ + # Required for libraries to be discoverable + pkgs.pkg-config + + # Required by patchable + pkgs.openssl + ]; } From 1366a0c4f1d8de97a3bc8714d9c77c78aba03ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 14 Feb 2025 17:16:04 +0100 Subject: [PATCH 42/48] Reword docs following @soenkeliebau's comments https://github.com/stackabletech/docker-images/pull/1003#discussion_r1956385473 --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c56b41e97..37cfc2368 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,9 @@ Many products apply Stackable-specific patches, managed by [Patchable](rust/patc Patchable doesn't _edit_ anything by itself. Instead, it's a uniform way to apply a set of patches to an upstream Git repository, and then export your local changes back into patch files. -You can then edit the branch created by patchable using any Git frontend, such as the git CLI or -[jj](https://jj-vcs.github.io/jj/latest/). + +It doesn't care about how you make your local changes - you can then edit the branch created by +patchable using any Git frontend, such as the git CLI or [jj](https://jj-vcs.github.io/jj/latest/). This way, the patch files are the global source of truth and track the history of our patch series, while you can still use the same familiar Git tools to manipulate them. From 908774478c16a361a8e6e903200061df40cdbb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 14 Feb 2025 17:28:05 +0100 Subject: [PATCH 43/48] Then begone --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 37cfc2368..974951782 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Many products apply Stackable-specific patches, managed by [Patchable](rust/patc Patchable doesn't _edit_ anything by itself. Instead, it's a uniform way to apply a set of patches to an upstream Git repository, and then export your local changes back into patch files. -It doesn't care about how you make your local changes - you can then edit the branch created by +It doesn't care about how you make your local changes - you can edit the branch created by patchable using any Git frontend, such as the git CLI or [jj](https://jj-vcs.github.io/jj/latest/). This way, the patch files are the global source of truth and track the history of our patch series, From c0f93d144a9c50479b11d59101f19438fd97f24d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 14 Feb 2025 17:41:48 +0100 Subject: [PATCH 44/48] Changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2dd14df2..1aad3e819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,12 +27,14 @@ All notable changes to this project will be documented in this file. - trino-cli: Add version 470 ([#999]). - trino-storage-connector: Add version 470 ([#999]). - superset: Add version `4.1.1` ([#991]). +- Added Patchable patch management tool ([#1003]). ### Changed - kafka: Bump 3.8.0 to 3.8.1 ([#995]). - Update registry references to oci ([#989]). - trino-storage-connector: Move the build out of trino/ for easier patching ([#996]). +- druid 26.0.0: Migrate to patchable ([#1003]). ### Removed @@ -71,6 +73,7 @@ All notable changes to this project will be documented in this file. [#997]: https://github.com/stackabletech/docker-images/pull/997 [#999]: https://github.com/stackabletech/docker-images/pull/999 [#1000]: https://github.com/stackabletech/docker-images/pull/1000 +[#1003]: https://github.com/stackabletech/docker-images/pull/1003 ## [24.11.1] - 2025-01-14 From 19af48c540ed4de9edf7059ef99a7f6036541076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Fri, 14 Feb 2025 17:44:03 +0100 Subject: [PATCH 45/48] Fix gitignore EOLs --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 75bf8c4e8..e1a295d51 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ __pycache__/ target/ # Patchable working files -patchable-work/ \ No newline at end of file +patchable-work/ From c1e039ce1b1cf38f3b87503e4d6d311c9e695a82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 17 Feb 2025 14:35:12 +0100 Subject: [PATCH 46/48] Update README.md Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73f4661ee..26f6c8838 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ cargo patchable export druid 26.0.0 git status ``` -> ![IMPORTANT] +> ![CAUTION] > `cargo patchable export` exports whatever is currently checked out (`HEAD`) in the worktree. > If you use `jj edit` (or `git switch`) then you _must_ go back to the tip before exporting, or > any patches after that point will be omitted from the export. From b2998f55aac8000aed2cb81b2bbe5b6b8e2d2dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 17 Feb 2025 14:35:19 +0100 Subject: [PATCH 47/48] Update README.md Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 26f6c8838..4e3b6f3d7 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ while you can still use the same familiar Git tools to manipulate them. pushd $(cargo patchable checkout druid 26.0.0) # Commit to add new patches +# NOTE: the commit message will be used to construct the patch filename. Spaces +# will be converted to hyphens automatically. git commit # Rebase against the base commit to edit or remove patches From 675d332a2e763bb208d527b0fabb23a13ec517b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Mon, 17 Feb 2025 15:01:43 +0100 Subject: [PATCH 48/48] Fix FMPP update patch metadata --- ...pp.patch => 0009-Update-FMPP-version.patch} | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) rename druid/stackable/patches/26.0.0/{09-update-fmpp.patch => 0009-Update-FMPP-version.patch} (51%) diff --git a/druid/stackable/patches/26.0.0/09-update-fmpp.patch b/druid/stackable/patches/26.0.0/0009-Update-FMPP-version.patch similarity index 51% rename from druid/stackable/patches/26.0.0/09-update-fmpp.patch rename to druid/stackable/patches/26.0.0/0009-Update-FMPP-version.patch index 3abb818da..67120aec1 100644 --- a/druid/stackable/patches/26.0.0/09-update-fmpp.patch +++ b/druid/stackable/patches/26.0.0/0009-Update-FMPP-version.patch @@ -1,8 +1,18 @@ -diff --git a/10-update-fmpp.patch b/10-update-fmpp.patch -new file mode 100644 -index 0000000000..e69de29bb2 +From 736165ab0fe73e0bef765f2cfd21cd800baddbc1 Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Thu, 12 Dec 2024 06:35:21 +0100 +Subject: Update FMPP version + +This is because FMPP Maven Plugin depends on FMPP in version 0.9.14 +which itself depends on a Freemarker version that has not been pinned. +Instead it specifies a "range" which resolves to a SNAPSHOT version +which we don't want. +--- + sql/pom.xml | 7 +++++++ + 1 file changed, 7 insertions(+) + diff --git a/sql/pom.xml b/sql/pom.xml -index bdd29f3f91..e5ba89f655 100644 +index e2bbd8c7f8..a72f96a6ca 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -322,6 +322,13 @@