From 0f83ab34f16a3ff11f808ef9f34aafffd2bfd55b Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Mon, 17 Nov 2025 17:13:19 +0100 Subject: [PATCH 1/6] PreserveFirstFoundResourceTransformer smoke test --- ...eserveFirstFoundResourceTransformerTest.kt | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PreserveFirstFoundResourceTransformerTest.kt diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PreserveFirstFoundResourceTransformerTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PreserveFirstFoundResourceTransformerTest.kt new file mode 100644 index 000000000..1efc0ff3e --- /dev/null +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PreserveFirstFoundResourceTransformerTest.kt @@ -0,0 +1,40 @@ +package com.github.jengelman.gradle.plugins.shadow.transformers + +import assertk.assertThat +import assertk.assertions.isEqualTo +import com.github.jengelman.gradle.plugins.shadow.testkit.containsOnly +import com.github.jengelman.gradle.plugins.shadow.testkit.getContent +import kotlin.io.path.appendText +import org.junit.jupiter.api.Test + +class PreserveFirstFoundResourceTransformerTest : BaseTransformerTest() { + @Test + fun smokeTest() { + val one = buildJarOne { + insert("dup", "content") + insert("foo", "content-foo") + } + val two = buildJarTwo { + insert("dup", "content-different") + insert("bar", "content-bar") + } + + projectScript.appendText( + transform( + dependenciesBlock = implementationFiles(one, two), + transformerBlock = """ + exclude("multiple-contents") + """.trimIndent(), + ), + ) + + runWithSuccess(shadowJarPath) + + assertThat(outputShadowedJar).useAll { + containsOnly("dup", "foo", "bar", "META-INF/", "META-INF/MANIFEST.MF") + getContent("dup").isEqualTo("content") + getContent("foo").isEqualTo("content-foo") + getContent("bar").isEqualTo("content-bar") + } + } +} From 81f60890577a6f690580e24ad49d20b681b93549 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Mon, 17 Nov 2025 09:54:38 +0100 Subject: [PATCH 2/6] `PropertiesFileTransformer` - add `MergeStrategy.Fail` --- api/shadow.api | 1 + docs/changes/README.md | 1 + .../PropertiesFileTransformerTest.kt | 36 ++++++++++++++----- .../transformers/PropertiesFileTransformer.kt | 20 +++++++++++ .../PropertiesFileTransformerTest.kt | 15 ++++++++ 5 files changed, 65 insertions(+), 8 deletions(-) diff --git a/api/shadow.api b/api/shadow.api index e6154e320..52c6fdbd0 100644 --- a/api/shadow.api +++ b/api/shadow.api @@ -434,6 +434,7 @@ public class com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesF public final class com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer$MergeStrategy : java/lang/Enum { public static final field Append Lcom/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer$MergeStrategy; public static final field Companion Lcom/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer$MergeStrategy$Companion; + public static final field Fail Lcom/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer$MergeStrategy; public static final field First Lcom/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer$MergeStrategy; public static final field Latest Lcom/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer$MergeStrategy; public static final fun from (Ljava/lang/String;)Lcom/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer$MergeStrategy; diff --git a/docs/changes/README.md b/docs/changes/README.md index 8113e7a5f..398d0ba29 100644 --- a/docs/changes/README.md +++ b/docs/changes/README.md @@ -11,6 +11,7 @@ - Expose `patternSet` of `ApacheNoticeResourceTransformer` as `public`. ([#1850](https://github.com/GradleUp/shadow/pull/1850)) - Expose `patternSet` of `PreserveFirstFoundResourceTransformer` as `public`. ([#1855](https://github.com/GradleUp/shadow/pull/1855)) - Support overriding output path of `ApacheNoticeResourceTransformer`. ([#1851](https://github.com/GradleUp/shadow/pull/1851)) +- Add new merge strategy 'fail' to `PropertiesFileTransformer`. ([#1856](https://github.com/GradleUp/shadow/pull/1856)) ### Changed diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt index b12c96175..abe6ec173 100644 --- a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt @@ -2,11 +2,15 @@ package com.github.jengelman.gradle.plugins.shadow.transformers import assertk.assertThat import assertk.assertions.contains +import assertk.assertions.containsSubList import assertk.assertions.isEqualTo +import assertk.assertions.isSameInstanceAs import com.github.jengelman.gradle.plugins.shadow.testkit.getContent import com.github.jengelman.gradle.plugins.shadow.transformers.PropertiesFileTransformer.MergeStrategy import com.github.jengelman.gradle.plugins.shadow.util.Issue import kotlin.io.path.appendText +import org.gradle.testkit.runner.TaskOutcome +import org.junit.jupiter.api.Assertions.fail import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.EnumSource @@ -33,15 +37,31 @@ class PropertiesFileTransformerTest : BaseTransformerTest() { ), ) - runWithSuccess(shadowJarPath) - - val expected = when (strategy) { - MergeStrategy.First -> arrayOf("key1=one", "key2=one", "key3=two") - MergeStrategy.Latest -> arrayOf("key1=one", "key2=two", "key3=two") - MergeStrategy.Append -> arrayOf("key1=one", "key2=one;two", "key3=two") + if (strategy == MergeStrategy.Fail) { + val run = runWithFailure(shadowJarPath) + val taskOutcome = run.task(":shadowJar")!! + assertThat(taskOutcome.outcome).isSameInstanceAs(TaskOutcome.FAILED) + assertThat(run.output.lines()).containsSubList( + listOf( + // Keep this list approach for Unix/Windows test compatibility. + "Execution failed for task ':shadowJar'.", + "> The following properties files have conflicting property values and cannot be merged:", + " * META-INF/test.properties", + " * Property key2 is duplicated 2 times with different values", + ), + ) + } else { + runWithSuccess(shadowJarPath) + + val expected = when (strategy) { + MergeStrategy.First -> arrayOf("key1=one", "key2=one", "key3=two") + MergeStrategy.Latest -> arrayOf("key1=one", "key2=two", "key3=two") + MergeStrategy.Append -> arrayOf("key1=one", "key2=one;two", "key3=two") + else -> fail("Unexpected strategy: $strategy") + } + val content = outputShadowedJar.use { it.getContent("META-INF/test.properties") } + assertThat(content).contains(*expected) } - val content = outputShadowedJar.use { it.getContent("META-INF/test.properties") } - assertThat(content).contains(*expected) } @Test diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt index aed3c054a..032bf7fb5 100644 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt @@ -11,6 +11,7 @@ import java.nio.charset.Charset import java.util.Properties import javax.inject.Inject import org.apache.tools.zip.ZipOutputStream +import org.gradle.api.GradleException import org.gradle.api.file.FileTreeElement import org.gradle.api.model.ObjectFactory import org.gradle.api.provider.MapProperty @@ -62,6 +63,9 @@ import org.gradle.api.tasks.Internal * - key2 = value2;balue2 * - key3 = value3 * + * With `mergeStrategy = MergeStrategy.Fail` the transformation will fail if there + * are conflicting values. + * * There are three additional properties that can be set: [paths], [mappings], * and [keyTransformer]. * The first contains a list of strings or regexes that will be used to determine if @@ -103,6 +107,9 @@ public open class PropertiesFileTransformer @Inject constructor( ) : ResourceTransformer { private inline val charset get() = Charset.forName(charsetName.get()) + @get:Internal + internal val conflicts: MutableMap> = mutableMapOf() + @get:Internal internal val propertiesEntries = mutableMapOf() @@ -156,6 +163,10 @@ public open class PropertiesFileTransformer @Inject constructor( props[key] = props.getProperty(key as String) + mergeSeparatorFor(context.path) + value } MergeStrategy.First -> Unit + MergeStrategy.Fail -> { + val conflictsForPath: MutableMap = conflicts.computeIfAbsent(context.path) { mutableMapOf() } + conflictsForPath.compute(key as String) { _, count -> (count ?: 1) + 1 } + } } } else { props[key] = value @@ -215,6 +226,14 @@ public open class PropertiesFileTransformer @Inject constructor( override fun hasTransformedResource(): Boolean = propertiesEntries.isNotEmpty() override fun modifyOutputStream(os: ZipOutputStream, preserveFileTimestamps: Boolean) { + if (conflicts.isNotEmpty()) { + throw GradleException( + "The following properties files have conflicting property values and cannot be merged:${conflicts + .map { (path, conflicts) -> "$path${conflicts.map { "Property ${it.key} is duplicated ${it.value} times with different values" }.joinToString(separator = "\n * ", prefix = "\n * ")}" } + .joinToString(separator = "\n * ", prefix = "\n * ")}", + ) + } + // Cannot close the writer as the OutputStream needs to remain open. val zipWriter = os.writer(charset) propertiesEntries.forEach { (path, props) -> @@ -231,6 +250,7 @@ public open class PropertiesFileTransformer @Inject constructor( First, Latest, Append, + Fail, ; public companion object { diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt index a51818829..ce8daec1f 100644 --- a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt @@ -64,6 +64,7 @@ class PropertiesFileTransformerTest : BaseTransformerTest, input2: Map, expectedOutput: Map, + expectedConflicts: Map>, ) { transformer.mergeStrategy.set(MergeStrategy.from(mergeStrategy)) transformer.mergeSeparator.set(mergeSeparator) @@ -74,6 +75,7 @@ class PropertiesFileTransformerTest : BaseTransformerTest>(), ), Arguments.of( "f.properties", @@ -271,6 +274,7 @@ class PropertiesFileTransformerTest : BaseTransformerTest>(), ), Arguments.of( "f.properties", @@ -279,6 +283,7 @@ class PropertiesFileTransformerTest : BaseTransformerTest>(), ), Arguments.of( "f.properties", @@ -287,6 +292,16 @@ class PropertiesFileTransformerTest : BaseTransformerTest>(), + ), + Arguments.of( + "f.properties", + "fail", + ";", + mapOf("foo" to "foo"), + mapOf("foo" to "bar"), + mapOf("foo" to "foo"), + mapOf("f.properties" to mapOf("foo" to 2)), ), ) From ef9ac07cc7142ae671f64d805d2cfed6d9757d4d Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Mon, 17 Nov 2025 10:10:41 +0100 Subject: [PATCH 3/6] `PropertiesFileTransformer` - make merged properties reproducible --- api/shadow.api | 1 + .../PropertiesFileTransformerTest.kt | 2 - .../shadow/internal/CleanProperties.kt | 31 ------- .../shadow/internal/ReproducibleProperties.kt | 60 ++++++++++++++ .../transformers/PropertiesFileTransformer.kt | 82 +++++++++---------- .../PropertiesFileTransformerTest.kt | 10 +-- 6 files changed, 106 insertions(+), 80 deletions(-) delete mode 100644 src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt create mode 100644 src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt diff --git a/api/shadow.api b/api/shadow.api index 52c6fdbd0..99d799409 100644 --- a/api/shadow.api +++ b/api/shadow.api @@ -419,6 +419,7 @@ public class com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesF public fun (Lorg/gradle/api/model/ObjectFactory;)V public fun canTransformResource (Lorg/gradle/api/file/FileTreeElement;)Z public fun getCharsetName ()Lorg/gradle/api/provider/Property; + public fun getEscapeUnicode ()Lorg/gradle/api/provider/Property; public fun getKeyTransformer ()Lkotlin/jvm/functions/Function1; public fun getMappings ()Lorg/gradle/api/provider/MapProperty; public fun getMergeSeparator ()Lorg/gradle/api/provider/Property; diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt index abe6ec173..d4c3d4362 100644 --- a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt @@ -171,8 +171,6 @@ class PropertiesFileTransformerTest : BaseTransformerTest() { val content = outputShadowedJar.use { it.getContent("META-INF/test.properties") } assertThat(content.trimIndent()).isEqualTo( """ - # - foo=one,two """.trimIndent(), ) diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt deleted file mode 100644 index 624aa422d..000000000 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt +++ /dev/null @@ -1,31 +0,0 @@ -package com.github.jengelman.gradle.plugins.shadow.internal - -import java.io.BufferedWriter -import java.io.IOException -import java.io.Writer -import java.util.Date -import java.util.Properties - -/** - * Introduced in order to remove prepended timestamp when creating output stream. - */ -internal class CleanProperties : Properties() { - @Throws(IOException::class) - override fun store(writer: Writer, comments: String) { - super.store(StripCommentsWithTimestampBufferedWriter(writer), comments) - } - - private class StripCommentsWithTimestampBufferedWriter(out: Writer) : BufferedWriter(out) { - private val lengthOfExpectedTimestamp = ("#" + Date().toString()).length - - @Throws(IOException::class) - override fun write(str: String) { - if (str.couldBeCommentWithTimestamp) return - super.write(str) - } - - private val String?.couldBeCommentWithTimestamp: Boolean get() { - return this != null && startsWith("#") && length == lengthOfExpectedTimestamp - } - } -} diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt new file mode 100644 index 000000000..b918eee6d --- /dev/null +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt @@ -0,0 +1,60 @@ +package com.github.jengelman.gradle.plugins.shadow.internal + +import java.io.OutputStream +import java.nio.charset.Charset +import java.util.HexFormat +import java.util.SortedMap + +/** + * Maintains a map of properties sorted by key, which can be written out to a text file with a consistent + * ordering to satisfy the requirements of reproducible builds. + */ +internal class ReproducibleProperties { + internal val props: SortedMap = sortedMapOf() + + fun writeProperties(charset: Charset, os: OutputStream, escape: Boolean) { + val zipWriter = os.writer(charset) + props.forEach { (key, value) -> + zipWriter.write(convertString(key, isKey = true, escape)) + zipWriter.write("=") + zipWriter.write(convertString(value, isKey = false, escape)) + zipWriter.write("\n") + } + zipWriter.flush() + } + + private fun convertString( + str: String, + isKey: Boolean, + escape: Boolean, + ): String { + val len = str.length + val out = StringBuilder() + val hex = HexFormat.of().withUpperCase() + for (x in 0.. 61) && (aChar.code < 127)) { + out.append(if (aChar == '\\') "\\\\" else aChar) + continue + } + when (aChar) { + ' ' -> { + if (x == 0 || isKey) out.append('\\') + out.append(' ') + } + '\t' -> out.append("\\t") + '\n' -> out.append("\\n") + '\r' -> out.append("\\r") + '\u000c' -> out.append("\\f") + '=', ':', '#', '!' -> out.append('\\').append(aChar) + else -> if (escape && ((aChar.code < 0x0020) || (aChar.code > 0x007e))) { + out.append("\\u").append(hex.toHexDigits(aChar)) + } else { + out.append(aChar) + } + } + } + return out.toString() + } +} diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt index 032bf7fb5..f7295a172 100644 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt @@ -1,6 +1,6 @@ package com.github.jengelman.gradle.plugins.shadow.transformers -import com.github.jengelman.gradle.plugins.shadow.internal.CleanProperties +import com.github.jengelman.gradle.plugins.shadow.internal.ReproducibleProperties import com.github.jengelman.gradle.plugins.shadow.internal.inputStream import com.github.jengelman.gradle.plugins.shadow.internal.mapProperty import com.github.jengelman.gradle.plugins.shadow.internal.property @@ -111,7 +111,7 @@ public open class PropertiesFileTransformer @Inject constructor( internal val conflicts: MutableMap> = mutableMapOf() @get:Internal - internal val propertiesEntries = mutableMapOf() + internal val propertiesEntries = mutableMapOf() @get:Input public open val paths: SetProperty = objectFactory.setProperty() @@ -125,6 +125,22 @@ public open class PropertiesFileTransformer @Inject constructor( @get:Input public open val mergeSeparator: Property = objectFactory.property(",") + /** + * Properties files are written without escaping unicode characters using the character set + * configured by [charsetName]. + * + * Set this property to `true` to escape all unicode characters in the properties file, producing + * ASCII compatible files. + */ + @get:Input + public open val escapeUnicode: Property = objectFactory.property(false) + + /** + * The character set to use when reading and writing property files. + * Defaults to `ISO-8859-1`. + * + * See also [escapeUnicode]. + */ @get:Input public open val charsetName: Property = objectFactory.property(Charsets.ISO_8859_1.name()) @@ -148,49 +164,35 @@ public open class PropertiesFileTransformer @Inject constructor( } override fun transform(context: TransformerContext) { - val props = propertiesEntries[context.path] - val incoming = loadAndTransformKeys(context.inputStream) - if (props == null) { - propertiesEntries[context.path] = incoming - } else { - for ((key, value) in incoming) { - if (props.containsKey(key)) { - when (MergeStrategy.from(mergeStrategyFor(context.path))) { - MergeStrategy.Latest -> { - props[key] = value - } - MergeStrategy.Append -> { - props[key] = props.getProperty(key as String) + mergeSeparatorFor(context.path) + value - } - MergeStrategy.First -> Unit - MergeStrategy.Fail -> { - val conflictsForPath: MutableMap = conflicts.computeIfAbsent(context.path) { mutableMapOf() } - conflictsForPath.compute(key as String) { _, count -> (count ?: 1) + 1 } - } + val props = propertiesEntries.computeIfAbsent(context.path) { ReproducibleProperties() }.props + val mergeStrategy = MergeStrategy.from(mergeStrategyFor(context.path)) + val mergeSeparator = if (mergeStrategy == MergeStrategy.Append) mergeSeparatorFor(context.path) else "" + loadAndTransformKeys(context.inputStream) { key, value -> + if (props.containsKey(key)) { + when (mergeStrategy) { + MergeStrategy.Latest -> { + props[key] = value + } + MergeStrategy.Append -> { + props[key] = props[key] + mergeSeparator + value + } + MergeStrategy.First -> Unit + MergeStrategy.Fail -> { + val conflictsForPath: MutableMap = conflicts.computeIfAbsent(context.path) { mutableMapOf() } + conflictsForPath.compute(key as String) { _, count -> (count ?: 1) + 1 } } - } else { - props[key] = value } + } else { + props[key] = value } } } - private fun loadAndTransformKeys(inputStream: InputStream): CleanProperties { - val props = CleanProperties() + private fun loadAndTransformKeys(inputStream: InputStream, keyValue: (key: String, value: String) -> Unit) { + val props = Properties() // InputStream closed by caller, so we don't do it here. props.load(inputStream.bufferedReader(charset)) - return transformKeys(props) - } - - private fun transformKeys(properties: Properties): CleanProperties { - if (keyTransformer == IDENTITY) { - return properties as CleanProperties - } - val result = CleanProperties() - properties.forEach { (key, value) -> - result[keyTransformer(key as String)] = value - } - return result + props.forEach { keyValue(keyTransformer(it.key as String), it.value as String) } } private fun mergeStrategyFor(path: String): String { @@ -235,13 +237,9 @@ public open class PropertiesFileTransformer @Inject constructor( } // Cannot close the writer as the OutputStream needs to remain open. - val zipWriter = os.writer(charset) propertiesEntries.forEach { (path, props) -> os.putNextEntry(zipEntry(path, preserveFileTimestamps)) - props.inputStream(charset).bufferedReader(charset).use { - it.copyTo(zipWriter) - } - zipWriter.flush() + props.writeProperties(charset, os, escapeUnicode.get()) os.closeEntry() } } diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt index ce8daec1f..7b492f9bf 100644 --- a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt @@ -74,7 +74,7 @@ class PropertiesFileTransformerTest : BaseTransformerTest Date: Sat, 15 Nov 2025 12:41:47 +0100 Subject: [PATCH 4/6] Add a convenience task to find resources in dependencies aka: "which dependencies contain the path XYZ" --- api/shadow.api | 19 ++++++ .../shadow/tasks/FindResourceInClasspath.kt | 63 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/FindResourceInClasspath.kt diff --git a/api/shadow.api b/api/shadow.api index 99d799409..f190426bb 100644 --- a/api/shadow.api +++ b/api/shadow.api @@ -176,6 +176,25 @@ public abstract class com/github/jengelman/gradle/plugins/shadow/tasks/Dependenc public fun resolve (Lorg/gradle/api/artifacts/Configuration;)Lorg/gradle/api/file/FileCollection; } +public abstract class com/github/jengelman/gradle/plugins/shadow/tasks/FindResourceInClasspath : org/gradle/api/DefaultTask, org/gradle/api/tasks/util/PatternFilterable { + public fun ()V + public fun (Lorg/gradle/api/tasks/util/PatternSet;)V + public synthetic fun (Lorg/gradle/api/tasks/util/PatternSet;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun exclude (Lgroovy/lang/Closure;)Lorg/gradle/api/tasks/util/PatternFilterable; + public fun exclude (Ljava/lang/Iterable;)Lorg/gradle/api/tasks/util/PatternFilterable; + public fun exclude (Lorg/gradle/api/specs/Spec;)Lorg/gradle/api/tasks/util/PatternFilterable; + public fun exclude ([Ljava/lang/String;)Lorg/gradle/api/tasks/util/PatternFilterable; + public abstract fun getClasspath ()Lorg/gradle/api/file/ConfigurableFileCollection; + public fun getExcludes ()Ljava/util/Set; + public fun getIncludes ()Ljava/util/Set; + public fun include (Lgroovy/lang/Closure;)Lorg/gradle/api/tasks/util/PatternFilterable; + public fun include (Ljava/lang/Iterable;)Lorg/gradle/api/tasks/util/PatternFilterable; + public fun include (Lorg/gradle/api/specs/Spec;)Lorg/gradle/api/tasks/util/PatternFilterable; + public fun include ([Ljava/lang/String;)Lorg/gradle/api/tasks/util/PatternFilterable; + public fun setExcludes (Ljava/lang/Iterable;)Lorg/gradle/api/tasks/util/PatternFilterable; + public fun setIncludes (Ljava/lang/Iterable;)Lorg/gradle/api/tasks/util/PatternFilterable; +} + public abstract interface class com/github/jengelman/gradle/plugins/shadow/tasks/InheritManifest : org/gradle/api/java/archives/Manifest { public fun inheritFrom ([Ljava/lang/Object;)V public abstract fun inheritFrom ([Ljava/lang/Object;Lorg/gradle/api/Action;)V diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/FindResourceInClasspath.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/FindResourceInClasspath.kt new file mode 100644 index 000000000..ed26ac1e8 --- /dev/null +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/tasks/FindResourceInClasspath.kt @@ -0,0 +1,63 @@ +package com.github.jengelman.gradle.plugins.shadow.tasks + +import org.gradle.api.DefaultTask +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.Classpath +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.TaskAction +import org.gradle.api.tasks.util.PatternFilterable +import org.gradle.api.tasks.util.PatternSet + +/** + * Helper task to temporarily add to your build script to find resources in the classpath that were + * identified as duplicates by [com.github.jengelman.gradle.plugins.shadow.transformers.MergePropertiesResourceTransformer] or + * [com.github.jengelman.gradle.plugins.shadow.transformers.DeduplicatingResourceTransformer]. + * + * First, add the task to your build script: + * ```kotlin + * val findResources by tasks.registering(FindResourceInClasspath::class) { + * // add configurations to search for resources in dependency jars + * classpath.from(configurations.runtimeClasspath) + * // the patterns to search for (it is a Gradle PatternFilterable) + * include( + * "META-INF/...", + * ) + * } + * ``` + * + * Then let `shadowJar` depend on the task, or just run the above task manually. + * + * ```kotlin + * tasks.named("shadowJar") { + * dependsOn(findResources) + * } + * ``` + */ +@Suppress("unused") +@CacheableTask +public abstract class FindResourceInClasspath(private val patternSet: PatternSet = PatternSet()) : + DefaultTask(), + PatternFilterable by patternSet { + @get:InputFiles + @get:Classpath + public abstract val classpath: ConfigurableFileCollection + + @Input + override fun getIncludes(): MutableSet = patternSet.includes + + @Input + override fun getExcludes(): MutableSet = patternSet.excludes + + @TaskAction + internal fun findResources() { + classpath.forEach { file -> + logger.lifecycle("scanning {}", file) + + project.zipTree(file).matching(patternSet).forEach { entry -> + logger.lifecycle(" -> {}", entry) + } + } + } +} From d9ce81a3369fa87172fb7c2a891b61210ad1a6d8 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 15 Nov 2025 12:42:17 +0100 Subject: [PATCH 5/6] Add transformer to deduplicate identical files based content --- api/shadow.api | 8 + .../DeduplicatingResourceTransformerTest.kt | 86 +++++++++++ .../DeduplicatingResourceTransformer.kt | 137 ++++++++++++++++++ .../PreserveFirstFoundResourceTransformer.kt | 6 + .../transformers/BaseTransformerTest.kt | 12 ++ .../DeduplicatingResourceTransformerTest.kt | 123 ++++++++++++++++ .../gradle/plugins/shadow/testkit/JarPath.kt | 29 ++++ 7 files changed, 401 insertions(+) create mode 100644 src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformerTest.kt create mode 100644 src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformer.kt create mode 100644 src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformerTest.kt diff --git a/api/shadow.api b/api/shadow.api index f190426bb..8ae497da2 100644 --- a/api/shadow.api +++ b/api/shadow.api @@ -327,6 +327,14 @@ public class com/github/jengelman/gradle/plugins/shadow/transformers/ComponentsX public final class com/github/jengelman/gradle/plugins/shadow/transformers/ComponentsXmlResourceTransformer$Companion { } +public class com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformer : com/github/jengelman/gradle/plugins/shadow/transformers/PatternFilterableResourceTransformer { + public fun (Lorg/gradle/api/model/ObjectFactory;Lorg/gradle/api/tasks/util/PatternSet;)V + public fun canTransformResource (Lorg/gradle/api/file/FileTreeElement;)Z + public final fun getObjectFactory ()Lorg/gradle/api/model/ObjectFactory; + public fun hasTransformedResource ()Z + public fun modifyOutputStream (Lorg/apache/tools/zip/ZipOutputStream;Z)V +} + public class com/github/jengelman/gradle/plugins/shadow/transformers/DontIncludeResourceTransformer : com/github/jengelman/gradle/plugins/shadow/transformers/ResourceTransformer { public fun (Lorg/gradle/api/model/ObjectFactory;)V public fun canTransformResource (Lorg/gradle/api/file/FileTreeElement;)Z diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformerTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformerTest.kt new file mode 100644 index 000000000..d115bc4a5 --- /dev/null +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformerTest.kt @@ -0,0 +1,86 @@ +package com.github.jengelman.gradle.plugins.shadow.transformers + +import assertk.assertThat +import assertk.assertions.any +import assertk.assertions.containsExactlyInAnyOrder +import assertk.assertions.containsSubList +import assertk.assertions.endsWith +import assertk.assertions.isEqualTo +import assertk.assertions.isSameInstanceAs +import com.github.jengelman.gradle.plugins.shadow.testkit.containsExactlyInAnyOrder +import com.github.jengelman.gradle.plugins.shadow.testkit.containsOnly +import com.github.jengelman.gradle.plugins.shadow.testkit.getContent +import com.github.jengelman.gradle.plugins.shadow.testkit.getContents +import kotlin.booleanArrayOf +import kotlin.io.path.appendText +import org.gradle.testkit.runner.TaskOutcome +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource + +class DeduplicatingResourceTransformerTest : BaseTransformerTest() { + @ParameterizedTest + @ValueSource(booleans = [false, true]) + fun conflictExclusion(excludeAll: Boolean) { + val one = buildJarOne { + insert("multiple-contents", "content") + insert("single-source", "content") + insert("same-content-twice", "content") + insert("differing-content-2", "content") + } + val two = buildJarTwo { + insert("multiple-contents", "content-is-different") + insert("same-content-twice", "content") + insert("differing-content-2", "content-is-different") + } + + projectScript.appendText( + transform( + dependenciesBlock = implementationFiles(one, two), + transformerBlock = """ + exclude("multiple-contents") + ${if (excludeAll) "exclude(\"differing-content-2\")" else ""} + """.trimIndent(), + ), + ) + + if (excludeAll) { + runWithSuccess(shadowJarPath) + assertThat(outputShadowedJar).useAll { + containsExactlyInAnyOrder( + // twice: + "multiple-contents", + "multiple-contents", + "single-source", + "same-content-twice", + // twice: + "differing-content-2", + "differing-content-2", + "META-INF/", + "META-INF/MANIFEST.MF", + ) + getContents("multiple-contents").containsExactlyInAnyOrder("content", "content-is-different") + getContent("single-source").isEqualTo("content") + getContent("same-content-twice").isEqualTo("content") + getContents("differing-content-2").containsExactlyInAnyOrder("content", "content-is-different") + } + } else { + val buildResult = runWithFailure(shadowJarPath) + assertThat(buildResult.task(":shadowJar")!!.outcome).isSameInstanceAs(TaskOutcome.FAILED) + val outputLines = buildResult.output.lines() + assertThat(outputLines).containsSubList( + listOf( + // Keep this list approach for Unix/Windows test compatibility. + "Execution failed for task ':shadowJar'.", + "> Found 1 path duplicate(s) with different content in the shadow JAR:", + " * differing-content-2", + ), + ) + assertThat(outputLines).any { + it.endsWith("/differing-content-2 (Hash: -1337566116240053116)") + } + assertThat(outputLines).any { + it.endsWith("/differing-content-2 (Hash: -6159701213549668473)") + } + } + } +} diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformer.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformer.kt new file mode 100644 index 000000000..e44518fdf --- /dev/null +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformer.kt @@ -0,0 +1,137 @@ +package com.github.jengelman.gradle.plugins.shadow.transformers + +import java.io.File +import java.nio.ByteBuffer +import java.security.MessageDigest +import javax.inject.Inject +import org.apache.tools.zip.ZipOutputStream +import org.gradle.api.GradleException +import org.gradle.api.file.FileTreeElement +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.util.PatternSet + +/** + * Transformer to include files with identical content only once in the shadow JAR. + * + * Multiple files with the same path but different content lead to an error. + * + * Some scenarios for duplicate resources in a shadow jar: + * * Duplicate `.class` files + * Having duplicate `.class` files with different is a situation indicating that the resulting jar is + * built with _incompatible_ classes, likely leading to issues during runtime. + * This situation can happen when one dependency is (also) included in an uber jar. + * * Duplicate `META-INF///pom.properties`/`xml` files. + * Some dependencies contain shaded variants of other dependencies. + * Tools that inspect jar files to extract the included dependencies, for example, for license auditing + * use cases or tools that collect information of all included dependencies, may rely on these files. + * Hence, it is desirable to retain the duplicate resource `pom.properties`/`xml` resources. + * + * `DeduplicatingResourceTransformer` checks all entries in the resulting jar. + * It is generally not recommended to use any of the [include] configuration functions. + * + * There are reasons to retain duplicate resources with different contents in the resulting jar. + * This can be achieved with the [exclude] configuration functions. + * + * To exclude a path or pattern from being deduplicated, for example, legit + * `META-INF///pom.properties`/`xml`, configure the transformer with an exclusion + * like the following: + * ```kotlin + * tasks.named("shadowJar").configure { + * // Keep pom.* files from different Guava versions in the jar. + * exclude("META-INF/maven/com.google.guava/guava/pom.*") + * // Duplicates with different content for all other resource paths will raise an error. + * } + * ``` + * + * *Tip*: the [com.github.jengelman.gradle.plugins.shadow.tasks.FindResourceInClasspath] convenience task + * can be used to find resources in a Gradle classpath/configuration. + * + * *Warning* Do **not** combine [PreserveFirstFoundResourceTransformer] with this transformer. + */ +@CacheableTransformer +public open class DeduplicatingResourceTransformer( + final override val objectFactory: ObjectFactory, + patternSet: PatternSet, +) : PatternFilterableResourceTransformer(patternSet) { + @get:Internal + internal val sources: MutableMap = mutableMapOf() + + @Inject + public constructor(objectFactory: ObjectFactory) : this(objectFactory, PatternSet()) + + internal data class PathInfos(val failOnDuplicateContent: Boolean) { + val filesPerHash: MutableMap> = mutableMapOf() + + fun uniqueContentCount() = filesPerHash.size + + fun addFile(hash: Long, file: File): Boolean { + var filesForHash: MutableList? = filesPerHash[hash] + val new = filesForHash == null + if (new) { + filesForHash = mutableListOf() + filesPerHash[hash] = filesForHash + } + filesForHash.add(file) + return new + } + } + + override fun canTransformResource(element: FileTreeElement): Boolean { + val file = element.file + val hash = hashForFile(file) + + val pathInfos = sources.computeIfAbsent(element.path) { PathInfos(patternSpec.isSatisfiedBy(element)) } + val retainInOutput = pathInfos.addFile(hash, file) + + return !retainInOutput + } + + override fun hasTransformedResource(): Boolean = true + + internal fun duplicateContentViolations(): Map = sources.filter { (_, pathInfos) -> pathInfos.failOnDuplicateContent && pathInfos.uniqueContentCount() > 1 } + + override fun modifyOutputStream(os: ZipOutputStream, preserveFileTimestamps: Boolean) { + val duplicatePaths = duplicateContentViolations() + + if (!duplicatePaths.isEmpty()) { + val message = + "Found ${duplicatePaths.size} path duplicate(s) with different content in the shadow JAR:" + + duplicatePaths + .map { (path, infos) -> + " * $path${infos.filesPerHash.map { (hash, files) -> + files.joinToString { file -> " * ${file.path} (Hash: $hash)" } + }.joinToString("\n", "\n", "")}" + } + .joinToString("\n", "\n", "") + throw GradleException(message) + } + } + + // Gradle's configuration uses Java serialization, which cannot serialize `MessageDigest` instances. + // Using a rather dirty mechanism to memoize the MD instance for task/transformer execution. + @Transient + private var digest: MessageDigest? = null + + internal fun hashForFile(file: File): Long { + if (digest == null) { + digest = MessageDigest.getInstance("SHA-256") + } + val d = digest!! + try { + file.inputStream().use { + val buffer = ByteArray(8192) + while (true) { + val rd = it.read(buffer) + if (rd == -1) { + break + } + d.update(buffer, 0, rd) + } + } + return ByteBuffer.wrap(d.digest()).getLong(0) + } catch (e: Exception) { + throw RuntimeException("Failed to read data or calculate hash for $file", e) + } + } +} diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PreserveFirstFoundResourceTransformer.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PreserveFirstFoundResourceTransformer.kt index 06d0e6a1b..eaac16bc7 100644 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PreserveFirstFoundResourceTransformer.kt +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PreserveFirstFoundResourceTransformer.kt @@ -18,6 +18,12 @@ import org.gradle.api.tasks.util.PatternSet * want to ensure that only the first found resource is included in the final JAR. If there are multiple resources with * the same path in a project and its dependencies, the first one found should be the project's. * + * This transformer deduplicates included resources based on the path name. + * See [DeduplicatingResourceTransformer] for a transformer that deduplicates based on the paths and contents of + * the resources. + * + * *Warning* Do **not** combine [DeduplicatingResourceTransformer] with this transformer. + * * @see [DuplicatesStrategy] * @see [ShadowJar.getDuplicatesStrategy] */ diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/BaseTransformerTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/BaseTransformerTest.kt index 89d4c2859..89b9fa51c 100644 --- a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/BaseTransformerTest.kt +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/BaseTransformerTest.kt @@ -6,6 +6,7 @@ import com.github.jengelman.gradle.plugins.shadow.testkit.requireResourceAsStrea import com.github.jengelman.gradle.plugins.shadow.transformers.ResourceTransformer.Companion.create import com.github.jengelman.gradle.plugins.shadow.util.noOpDelegate import com.github.jengelman.gradle.plugins.shadow.util.testObjectFactory +import java.io.File import java.lang.reflect.ParameterizedType import java.util.Locale import kotlin.io.path.createTempFile @@ -14,6 +15,7 @@ import org.apache.tools.zip.ZipOutputStream import org.gradle.api.file.FileTreeElement import org.gradle.api.file.RelativePath import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.io.TempDir abstract class BaseTransformerTest { protected lateinit var transformer: T @@ -41,6 +43,16 @@ abstract class BaseTransformerTest { return canTransformResource(element) } + fun ResourceTransformer.canTransformResource(path: String, file: File): Boolean { + val element = object : FileTreeElement by noOpDelegate() { + private val _relativePath = RelativePath.parse(true, path) + override fun getPath(): String = _relativePath.pathString + override fun getRelativePath(): RelativePath = _relativePath + override fun getFile(): File = file + } + return canTransformResource(element) + } + fun JarPath.readContentLines(resourceName: String = MANIFEST_NAME): List { return use { it.getStream(resourceName).bufferedReader().readLines() } } diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformerTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformerTest.kt new file mode 100644 index 000000000..c4cd661b6 --- /dev/null +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/DeduplicatingResourceTransformerTest.kt @@ -0,0 +1,123 @@ +package com.github.jengelman.gradle.plugins.shadow.transformers + +import assertk.assertThat +import assertk.assertions.containsExactlyInAnyOrder +import assertk.assertions.containsOnly +import assertk.assertions.isEqualTo +import assertk.assertions.isFalse +import assertk.assertions.isNotEqualTo +import assertk.assertions.isTrue +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource + +class DeduplicatingResourceTransformerTest : BaseTransformerTest() { + + @TempDir + lateinit var tempDir: Path + + lateinit var file1: File + lateinit var file2: File + lateinit var file3: File + + var hash1: Long = 0L + var hash2: Long = 0L + var hash3: Long = 0L + + @BeforeEach + fun setupFiles() { + val file1 = tempDir.resolve("file1") + val file2 = tempDir.resolve("file2") + val file3 = tempDir.resolve("file3") + + val content1 = "content1".toByteArray() + val content2 = "content2".toByteArray() + + Files.write(file1, content1) + Files.write(file2, content1) + Files.write(file3, content2) + + this.file1 = file1.toFile() + this.file2 = file2.toFile() + this.file3 = file3.toFile() + + this.hash1 = transformer.hashForFile(this.file1) + this.hash2 = transformer.hashForFile(this.file2) + this.hash3 = transformer.hashForFile(this.file3) + } + + @Test + fun hashing() { + assertThat(hash1).isEqualTo(hash2) + assertThat(hash1).isNotEqualTo(hash3) + } + + @ParameterizedTest + @ValueSource(booleans = [false, true]) + fun duplicateContent(exclusionCheck: Boolean) { + with(transformer) { + if (!exclusionCheck) { + exclude("multiple-contents") + } + + // new path, new file content --> retain resource + assertThat(canTransformResource("multiple-contents", file1)).isFalse() + // same path, same file content --> skip resource + assertThat(canTransformResource("multiple-contents", file2)).isTrue() + // same path, different file content --> retain resource (even if it's a duplicate) + assertThat(canTransformResource("multiple-contents", file3)).isFalse() + + assertThat(canTransformResource("single-source", file1)).isFalse() + + assertThat(canTransformResource("same-content-twice", file1)).isFalse() + assertThat(canTransformResource("same-content-twice", file2)).isTrue() + + assertThat(canTransformResource("differing-content-2", file1)).isFalse() + assertThat(canTransformResource("differing-content-2", file3)).isFalse() + + assertThat(sources.keys).containsExactlyInAnyOrder( + "multiple-contents", + "single-source", + "same-content-twice", + "differing-content-2", + ) + + val pathInfosMultipleContents = sources["multiple-contents"]!! + assertThat(pathInfosMultipleContents.failOnDuplicateContent).isEqualTo(exclusionCheck) + assertThat(pathInfosMultipleContents.uniqueContentCount()).isEqualTo(2) + assertThat(pathInfosMultipleContents.filesPerHash).containsOnly( + hash1 to listOf(file1, file2), + hash3 to listOf(file3), + ) + + val pathInfosSingleSource = sources["single-source"]!! + assertThat(pathInfosSingleSource.failOnDuplicateContent).isTrue() + assertThat(pathInfosSingleSource.uniqueContentCount()).isEqualTo(1) + assertThat(pathInfosSingleSource.filesPerHash).containsOnly(hash1 to listOf(file1)) + + val pathInfosSameContentTwice = sources["same-content-twice"]!! + assertThat(pathInfosSameContentTwice.failOnDuplicateContent).isTrue() + assertThat(pathInfosSameContentTwice.uniqueContentCount()).isEqualTo(1) + assertThat(pathInfosSameContentTwice.filesPerHash).containsOnly(hash1 to listOf(file1, file2)) + + val pathInfosDifferingContent2 = sources["differing-content-2"]!! + assertThat(pathInfosDifferingContent2.failOnDuplicateContent).isTrue() + assertThat(pathInfosDifferingContent2.uniqueContentCount()).isEqualTo(2) + assertThat(pathInfosDifferingContent2.filesPerHash).containsOnly(hash1 to listOf(file1), hash3 to listOf(file3)) + + if (exclusionCheck) { + assertThat(duplicateContentViolations()).containsOnly( + "multiple-contents" to pathInfosMultipleContents, + "differing-content-2" to pathInfosDifferingContent2, + ) + } else { + assertThat(duplicateContentViolations()).containsOnly("differing-content-2" to pathInfosDifferingContent2) + } + } + } +} diff --git a/src/testKit/kotlin/com/github/jengelman/gradle/plugins/shadow/testkit/JarPath.kt b/src/testKit/kotlin/com/github/jengelman/gradle/plugins/shadow/testkit/JarPath.kt index a213bc939..26285de18 100644 --- a/src/testKit/kotlin/com/github/jengelman/gradle/plugins/shadow/testkit/JarPath.kt +++ b/src/testKit/kotlin/com/github/jengelman/gradle/plugins/shadow/testkit/JarPath.kt @@ -2,11 +2,14 @@ package com.github.jengelman.gradle.plugins.shadow.testkit import assertk.Assert import assertk.assertions.containsAtLeast +import assertk.assertions.containsExactlyInAnyOrder import assertk.assertions.containsNone import assertk.assertions.containsOnly import java.io.InputStream +import java.nio.file.Files import java.nio.file.Path import java.util.jar.JarFile +import java.util.jar.JarInputStream import java.util.zip.ZipFile /** @@ -44,6 +47,26 @@ fun ZipFile.getStream(entryName: String): InputStream { fun Assert.getContent(entryName: String) = transform { it.getContent(entryName) } +/** + * Scans the jar file for all entries that match the specified [entryName], + * [getContent] or [getStream] return only one of the matching entries; + * which one these functions return is undefined. + */ +fun Assert.getContents(entryName: String) = transform { + Files.newInputStream(it.path).use { + val jarInput = JarInputStream(it) + val contents = mutableListOf() + while (true) { + val entry = jarInput.nextEntry ?: break + if (entry.name == entryName) { + contents.add(jarInput.readAllBytes().toString(Charsets.UTF_8)) + } + jarInput.closeEntry() + } + contents + } +} + fun Assert.getMainAttr(name: String) = transform { it.getMainAttr(name) } /** @@ -64,6 +87,12 @@ fun Assert.containsNone(vararg entries: String) = toEntries().containsN */ fun Assert.containsOnly(vararg entries: String) = toEntries().containsOnly(*entries) +/** + * Ensures the JAR contains only the specified entries. + * Used alone, without [containsAtLeast] or [containsNone]. + */ +fun Assert.containsExactlyInAnyOrder(vararg entries: String) = toEntries().containsExactlyInAnyOrder(*entries) + private fun Assert.toEntries() = transform { actual -> actual.entries().toList().map { it.name } } From 6b9117883e8b874f35967e396c2c08678fe83b54 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 15 Nov 2025 12:42:45 +0100 Subject: [PATCH 6/6] Add transformer to merge licenses into a single license file --- api/shadow.api | 15 ++ .../MergeLicenseResourceTransformerTest.kt | 61 +++++++ .../MergeLicenseResourceTransformer.kt | 171 ++++++++++++++++++ .../MergeLicenseResourceTransformerTest.kt | 135 ++++++++++++++ 4 files changed, 382 insertions(+) create mode 100644 src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformerTest.kt create mode 100644 src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformer.kt create mode 100644 src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformerTest.kt diff --git a/api/shadow.api b/api/shadow.api index 8ae497da2..46497c0d7 100644 --- a/api/shadow.api +++ b/api/shadow.api @@ -409,6 +409,21 @@ public class com/github/jengelman/gradle/plugins/shadow/transformers/ManifestRes public fun transform (Lcom/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext;)V } +public class com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformer : com/github/jengelman/gradle/plugins/shadow/transformers/PatternFilterableResourceTransformer { + public fun (Lorg/gradle/api/model/ObjectFactory;)V + public fun (Lorg/gradle/api/model/ObjectFactory;Lorg/gradle/api/tasks/util/PatternSet;)V + public final fun getArtifactLicense ()Lorg/gradle/api/file/RegularFileProperty; + public final fun getArtifactLicenseSpdxId ()Lorg/gradle/api/provider/Property; + public fun getExcludes ()Ljava/util/Set; + public final fun getFirstSeparator ()Lorg/gradle/api/provider/Property; + public fun getIncludes ()Ljava/util/Set; + public final fun getOutputPath ()Lorg/gradle/api/provider/Property; + public final fun getSeparator ()Lorg/gradle/api/provider/Property; + public fun hasTransformedResource ()Z + public fun modifyOutputStream (Lorg/apache/tools/zip/ZipOutputStream;Z)V + public fun transform (Lcom/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext;)V +} + public abstract class com/github/jengelman/gradle/plugins/shadow/transformers/PatternFilterableResourceTransformer : com/github/jengelman/gradle/plugins/shadow/transformers/ResourceTransformer, org/gradle/api/tasks/util/PatternFilterable { public fun (Lorg/gradle/api/tasks/util/PatternSet;)V public fun canTransformResource (Lorg/gradle/api/file/FileTreeElement;)Z diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformerTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformerTest.kt new file mode 100644 index 000000000..79560e560 --- /dev/null +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformerTest.kt @@ -0,0 +1,61 @@ +package com.github.jengelman.gradle.plugins.shadow.transformers + +import assertk.assertThat +import assertk.assertions.contains +import assertk.assertions.containsExactly +import assertk.assertions.isEqualTo +import com.github.jengelman.gradle.plugins.shadow.testkit.containsExactlyInAnyOrder +import com.github.jengelman.gradle.plugins.shadow.testkit.getContent +import kotlin.io.path.appendText +import kotlin.io.path.writeText +import kotlin.text.lines +import org.junit.jupiter.api.Test + +class MergeLicenseResourceTransformerTest : BaseTransformerTest() { + + @Test + fun twoLicenses() { + val one = buildJarOne { + insert("META-INF/LICENSE", "license one") + } + val two = buildJarTwo { + insert("META-INF/LICENSE", "license two") + } + + val artifactLicense = projectRoot.resolve("my-license") + artifactLicense.writeText("artifact license text") + + projectScript.appendText( + transform( + dependenciesBlock = implementationFiles(one, two), + transformerBlock = """ + outputPath = 'MY_LICENSE' + artifactLicense = file('my-license') + firstSeparator = '####' + separator = '----' + """.trimIndent(), + ), + ) + + runWithSuccess(shadowJarPath) + + assertThat(outputShadowedJar).useAll { + containsExactlyInAnyOrder( + "MY_LICENSE", + "META-INF/", + "META-INF/MANIFEST.MF", + ) + getContent("MY_LICENSE").given { + assertThat(it.lines()).isEqualTo( + listOf( + "artifact license text", + "####", + "license one", + "----", + "license two", + ), + ) + } + } + } +} diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformer.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformer.kt new file mode 100644 index 000000000..432b1aaa0 --- /dev/null +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformer.kt @@ -0,0 +1,171 @@ +package com.github.jengelman.gradle.plugins.shadow.transformers + +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowCopyAction +import java.io.OutputStream +import java.nio.charset.StandardCharsets.UTF_8 +import java.util.LinkedHashSet +import javax.inject.Inject +import org.apache.tools.zip.ZipEntry +import org.apache.tools.zip.ZipOutputStream +import org.gradle.api.file.FileTreeElement +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.util.PatternSet + +/** + * Generates a license file using the configured license text source. + * + * A mandatory `SPDX-License-Identifier` is placed in front of the license text to avoid ambiguous + * license detection by license-detection-tools. + * + * License texts found in the files names `META-INF/LICENSE`, `META-INF/LICENSE.txt`, + * `META-INF/LICENSE.md`, `LICENSE`, `LICENSE.txt`, `LICENSE.md` are included from the shadow jar + * sources. Use the [PatternFilterable][org.gradle.api.tasks.util.PatternFilterable] functions to + * specify a different set of files to include, the paths mentioned above are then not considered + * unless explicitly included. + */ +@Suppress("unused") +@CacheableTransformer +public open class MergeLicenseResourceTransformer( + objectFactory: ObjectFactory, + patternSet: PatternSet, +) : PatternFilterableResourceTransformer(patternSet) { + private val initializePatternSet by lazy { + if (patternSet.isEmpty) { + includeDefaults() + } + } + + @get:Internal + internal val elements: MutableSet = LinkedHashSet() + + public fun includeDefaults(): MergeLicenseResourceTransformer { + patternSet.include( + "META-INF/LICENSE", + "META-INF/LICENSE.txt", + "META-INF/LICENSE.md", + "LICENSE", + "LICENSE.txt", + "LICENSE.md", + ) + return this + } + + @Input + override fun getIncludes(): MutableSet = patternSet.includes + + @Input + override fun getExcludes(): MutableSet = patternSet.excludes + + /** Path to write the aggregated license file to. Defaults to `META-INF/LICENSE`. */ + @get:Input + public val outputPath: Property = + objectFactory.property(String::class.java).value("META-INF/LICENSE") + + /** + * The generated license file is potentially a collection of multiple license texts. To avoid + * ambiguous license detection by license-detection-tools, an SPDX license identifier header + * (`SPDX-License-Identifier:`) is added at the beginning of the generated file if the value of + * this property is present and not empty. Defaults to `Apache-2.0`. + */ + @get:Input + public val artifactLicenseSpdxId: Property = + objectFactory.property(String::class.java).value("Apache-2.0") + + /** Path to the project's license text, this property *must* be configured. */ + @get:InputFile + @get:PathSensitive(PathSensitivity.RELATIVE) + public val artifactLicense: RegularFileProperty = objectFactory.fileProperty() + + /** + * Separator between the project's license text and license texts from the included dependencies. + */ + @get:Input + public val firstSeparator: Property = + objectFactory + .property(String::class.java) + .value( + """ + + ${"-".repeat(120)} + + This artifact includes dependencies with the following licenses: + ---------------------------------------------------------------- + + """ + .trimIndent(), + ) + + /** Separator between included dependency license texts. */ + @get:Input + public val separator: Property = + objectFactory.property(String::class.java).value("\n${"-".repeat(120)}\n") + + @Inject + public constructor(objectFactory: ObjectFactory) : this( + objectFactory, + patternSet = PatternSet(), + ) + + override fun canTransformResource(element: FileTreeElement): Boolean { + // Init once before patternSpec is accessed. + initializePatternSet + return super.canTransformResource(element) + } + + override fun transform(context: TransformerContext) { + transformInternal(context.inputStream.readAllBytes()) + } + + internal fun transformInternal(bytes: ByteArray) { + val content = bytes.toString(UTF_8).trim('\n', '\r') + if (!content.isEmpty()) { + elements.add(content) + } + } + + override fun hasTransformedResource(): Boolean = true + + override fun modifyOutputStream(os: ZipOutputStream, preserveFileTimestamps: Boolean) { + os.putNextEntry( + ZipEntry(outputPath.get()).apply { time = ShadowCopyAction.CONSTANT_TIME_FOR_ZIP_ENTRIES }, + ) + + writeLicenseFile(os) + + os.closeEntry() + } + + internal fun writeLicenseFile(os: OutputStream) { + if (artifactLicenseSpdxId.isPresent) { + val spdxId = artifactLicenseSpdxId.get() + if (spdxId.isBlank()) { + os.write("SPDX-License-Identifier: $spdxId\n".toByteArray(UTF_8)) + } + } + os.write(artifactLicense.get().asFile.readBytes()) + + if (!elements.isEmpty()) { + os.write("\n".toByteArray(UTF_8)) + os.write(firstSeparator.get().toByteArray(UTF_8)) + os.write("\n".toByteArray(UTF_8)) + + var first = true + val separator = (this.separator.get() + "\n").toByteArray(UTF_8) + for (element in elements) { + if (!first) { + os.write("\n".toByteArray(UTF_8)) + os.write(separator) + } + os.write(element.toByteArray(UTF_8)) + first = false + } + } + } +} diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformerTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformerTest.kt new file mode 100644 index 000000000..20485d90d --- /dev/null +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/MergeLicenseResourceTransformerTest.kt @@ -0,0 +1,135 @@ +package com.github.jengelman.gradle.plugins.shadow.transformers + +import assertk.assertThat +import assertk.assertions.containsExactlyInAnyOrder +import assertk.assertions.isEmpty +import assertk.assertions.isEqualTo +import assertk.assertions.isFalse +import assertk.assertions.isNotEqualTo +import assertk.assertions.isTrue +import java.io.ByteArrayOutputStream +import java.nio.file.Path +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir + +class MergeLicenseResourceTransformerTest : BaseTransformerTest() { + @Test + fun defaultIncludes() { + with(transformer) { + assertThat(canTransformResource("META-INF/LICENSE")).isTrue() + assertThat(canTransformResource("META-INF/LICENSE.txt")).isTrue() + assertThat(canTransformResource("META-INF/LICENSE.md")).isTrue() + assertThat(canTransformResource("LICENSE")).isTrue() + assertThat(canTransformResource("LICENSE.txt")).isTrue() + assertThat(canTransformResource("LICENSE.md")).isTrue() + assertThat(canTransformResource("something else")).isFalse() + } + } + + @Test + fun customIncludes() { + with(transformer) { + include("META-INF/FOO") + assertThat(canTransformResource("META-INF/FOO")).isTrue() + assertThat(canTransformResource("META-INF/LICENSE")).isFalse() + assertThat(canTransformResource("META-INF/LICENSE.txt")).isFalse() + assertThat(canTransformResource("META-INF/LICENSE.md")).isFalse() + assertThat(canTransformResource("LICENSE")).isFalse() + assertThat(canTransformResource("LICENSE.txt")).isFalse() + assertThat(canTransformResource("LICENSE.md")).isFalse() + assertThat(canTransformResource("something else")).isFalse() + } + } + + @Test + fun customIncludesWithDefaults() { + with(transformer) { + include("META-INF/FOO") + includeDefaults() + assertThat(canTransformResource("META-INF/FOO")).isTrue() + assertThat(canTransformResource("META-INF/LICENSE")).isTrue() + assertThat(canTransformResource("META-INF/LICENSE.txt")).isTrue() + assertThat(canTransformResource("META-INF/LICENSE.md")).isTrue() + assertThat(canTransformResource("LICENSE")).isTrue() + assertThat(canTransformResource("LICENSE.txt")).isTrue() + assertThat(canTransformResource("LICENSE.md")).isTrue() + assertThat(canTransformResource("something else")).isFalse() + } + } + + @Test + fun deduplicateLicenseTexts(@TempDir tempDir: Path) { + with(transformer) { + transformInternal("license one".toByteArray()) + transformInternal("\r\nlicense one\r\n".toByteArray()) + transformInternal("\nlicense one\n".toByteArray()) + transformInternal(" license two".toByteArray()) + transformInternal("\r\n\n\r\n\n license two".toByteArray()) + transformInternal(" license two\r\n\n\r\n\n".toByteArray()) + transformInternal("license three".toByteArray()) + + val artifactLicenseFile = tempDir.resolve("artifact-license").toFile() + artifactLicenseFile.writeText("artifact license file content") + artifactLicense.set(artifactLicenseFile) + + assertThat(elements).containsExactlyInAnyOrder("license one", " license two", "license three") + + val baos = ByteArrayOutputStream() + writeLicenseFile(baos) + val written = baos.toByteArray().toString(Charsets.UTF_8).lines() + assertThat(written).isEqualTo( + listOf( + "artifact license file content", + ) + firstSeparator.get().lines() + + "license one" + + separator.get().lines() + + " license two" + + separator.get().lines() + + "license three", + ) + } + } + + @Test + fun singleAdditionalLicense(@TempDir tempDir: Path) { + with(transformer) { + transformInternal("license one".toByteArray()) + + val artifactLicenseFile = tempDir.resolve("artifact-license").toFile() + artifactLicenseFile.writeText("artifact license file content") + artifactLicense.set(artifactLicenseFile) + + assertThat(elements).containsExactlyInAnyOrder("license one") + + val baos = ByteArrayOutputStream() + writeLicenseFile(baos) + val written = baos.toByteArray().toString(Charsets.UTF_8).lines() + assertThat(written).isEqualTo( + listOf( + "artifact license file content", + ) + firstSeparator.get().lines() + + "license one", + ) + } + } + + @Test + fun noAdditionalLicenses(@TempDir tempDir: Path) { + with(transformer) { + val artifactLicenseFile = tempDir.resolve("artifact-license").toFile() + artifactLicenseFile.writeText("artifact license file content") + artifactLicense.set(artifactLicenseFile) + + assertThat(elements).isEmpty() + + val baos = ByteArrayOutputStream() + writeLicenseFile(baos) + val written = baos.toByteArray().toString(Charsets.UTF_8).lines() + assertThat(written).isEqualTo( + listOf( + "artifact license file content", + ), + ) + } + } +}