From 51a1215fbbdbc5951928ee270964194e08d64d48 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Mon, 17 Nov 2025 10:10:41 +0100 Subject: [PATCH 1/3] `PropertiesFileTransformer` - make merged properties reproducible `PropertiesFileTransformer` leverages `java.util.Properties`, which relies on `java.util.Hashtable`. The serialized properties are not guaranteed to be reproducible. This change changes the transformer to use a sorted map to generate reproducible output. The existing class `CleanProperties` is replaced with `ReproducibleProperties`. Functionality around charset handling is retained, and extended with a functionality to generate unicode escapes (ASCII output). --- api/shadow.api | 1 + docs/changes/README.md | 1 + .../PropertiesFileTransformerTest.kt | 2 - .../shadow/internal/CleanProperties.kt | 31 ------- .../shadow/internal/ReproducibleProperties.kt | 60 ++++++++++++++ .../transformers/PropertiesFileTransformer.kt | 83 +++++++++---------- .../PropertiesFileTransformerTest.kt | 10 +-- 7 files changed, 107 insertions(+), 81 deletions(-) delete mode 100644 src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt create mode 100644 src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt diff --git a/api/shadow.api b/api/shadow.api index ecb12177c..d633bb907 100644 --- a/api/shadow.api +++ b/api/shadow.api @@ -439,6 +439,7 @@ public class com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesF public fun (Lorg/gradle/api/model/ObjectFactory;)V public fun canTransformResource (Lorg/gradle/api/file/FileTreeElement;)Z public fun getCharsetName ()Lorg/gradle/api/provider/Property; + public fun getEscapeUnicode ()Lorg/gradle/api/provider/Property; public fun getKeyTransformer ()Lkotlin/jvm/functions/Function1; public fun getMappings ()Lorg/gradle/api/provider/MapProperty; public fun getMergeSeparator ()Lorg/gradle/api/provider/Property; diff --git a/docs/changes/README.md b/docs/changes/README.md index ccdf59322..40820254a 100644 --- a/docs/changes/README.md +++ b/docs/changes/README.md @@ -20,6 +20,7 @@ - Update ASM and jdependency to support Java 26. ([#1799](https://github.com/GradleUp/shadow/pull/1799)) - Bump min Gradle requirement to 9.0.0. ([#1801](https://github.com/GradleUp/shadow/pull/1801)) - Deprecate `PreserveFirstFoundResourceTransformer.resources`. ([#1855](https://github.com/GradleUp/shadow/pull/1855)) +- Make the output of `PropertiesFileTransformer` reproducible. ([#1861](https://github.com/GradleUp/shadow/pull/1861)) ### Fixed diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt index ae8802f31..ac392e43d 100644 --- a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt @@ -168,8 +168,6 @@ class PropertiesFileTransformerTest : BaseTransformerTest() { val content = outputShadowedJar.use { it.getContent("META-INF/test.properties") } assertThat(content.trimIndent()).isEqualTo( """ - # - foo=one,two """.trimIndent(), ) diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt deleted file mode 100644 index 624aa422d..000000000 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt +++ /dev/null @@ -1,31 +0,0 @@ -package com.github.jengelman.gradle.plugins.shadow.internal - -import java.io.BufferedWriter -import java.io.IOException -import java.io.Writer -import java.util.Date -import java.util.Properties - -/** - * Introduced in order to remove prepended timestamp when creating output stream. - */ -internal class CleanProperties : Properties() { - @Throws(IOException::class) - override fun store(writer: Writer, comments: String) { - super.store(StripCommentsWithTimestampBufferedWriter(writer), comments) - } - - private class StripCommentsWithTimestampBufferedWriter(out: Writer) : BufferedWriter(out) { - private val lengthOfExpectedTimestamp = ("#" + Date().toString()).length - - @Throws(IOException::class) - override fun write(str: String) { - if (str.couldBeCommentWithTimestamp) return - super.write(str) - } - - private val String?.couldBeCommentWithTimestamp: Boolean get() { - return this != null && startsWith("#") && length == lengthOfExpectedTimestamp - } - } -} diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt new file mode 100644 index 000000000..b918eee6d --- /dev/null +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt @@ -0,0 +1,60 @@ +package com.github.jengelman.gradle.plugins.shadow.internal + +import java.io.OutputStream +import java.nio.charset.Charset +import java.util.HexFormat +import java.util.SortedMap + +/** + * Maintains a map of properties sorted by key, which can be written out to a text file with a consistent + * ordering to satisfy the requirements of reproducible builds. + */ +internal class ReproducibleProperties { + internal val props: SortedMap = sortedMapOf() + + fun writeProperties(charset: Charset, os: OutputStream, escape: Boolean) { + val zipWriter = os.writer(charset) + props.forEach { (key, value) -> + zipWriter.write(convertString(key, isKey = true, escape)) + zipWriter.write("=") + zipWriter.write(convertString(value, isKey = false, escape)) + zipWriter.write("\n") + } + zipWriter.flush() + } + + private fun convertString( + str: String, + isKey: Boolean, + escape: Boolean, + ): String { + val len = str.length + val out = StringBuilder() + val hex = HexFormat.of().withUpperCase() + for (x in 0.. 61) && (aChar.code < 127)) { + out.append(if (aChar == '\\') "\\\\" else aChar) + continue + } + when (aChar) { + ' ' -> { + if (x == 0 || isKey) out.append('\\') + out.append(' ') + } + '\t' -> out.append("\\t") + '\n' -> out.append("\\n") + '\r' -> out.append("\\r") + '\u000c' -> out.append("\\f") + '=', ':', '#', '!' -> out.append('\\').append(aChar) + else -> if (escape && ((aChar.code < 0x0020) || (aChar.code > 0x007e))) { + out.append("\\u").append(hex.toHexDigits(aChar)) + } else { + out.append(aChar) + } + } + } + return out.toString() + } +} diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt index d938cdb1a..d0b2e15da 100644 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt @@ -1,7 +1,6 @@ package com.github.jengelman.gradle.plugins.shadow.transformers -import com.github.jengelman.gradle.plugins.shadow.internal.CleanProperties -import com.github.jengelman.gradle.plugins.shadow.internal.inputStream +import com.github.jengelman.gradle.plugins.shadow.internal.ReproducibleProperties import com.github.jengelman.gradle.plugins.shadow.internal.mapProperty import com.github.jengelman.gradle.plugins.shadow.internal.property import com.github.jengelman.gradle.plugins.shadow.internal.setProperty @@ -109,7 +108,7 @@ public open class PropertiesFileTransformer @Inject constructor( internal val conflicts: MutableMap> = mutableMapOf() @get:Internal - internal val propertiesEntries = mutableMapOf() + internal val propertiesEntries = mutableMapOf() @get:Input public open val paths: SetProperty = objectFactory.setProperty() @@ -123,6 +122,22 @@ public open class PropertiesFileTransformer @Inject constructor( @get:Input public open val mergeSeparator: Property = objectFactory.property(",") + /** + * Properties files are written without escaping Unicode characters using the character set + * configured by [charsetName]. + * + * Set this property to `true` to escape all Unicode characters in the properties file, producing + * ASCII compatible files. + */ + @get:Input + public open val escapeUnicode: Property = objectFactory.property(false) + + /** + * The character set to use when reading and writing property files. + * Defaults to `ISO-8859-1`. + * + * See also [escapeUnicode]. + */ @get:Input public open val charsetName: Property = objectFactory.property(Charsets.ISO_8859_1.name()) @@ -146,49 +161,35 @@ public open class PropertiesFileTransformer @Inject constructor( } override fun transform(context: TransformerContext) { - val props = propertiesEntries[context.path] - val incoming = loadAndTransformKeys(context.inputStream) - if (props == null) { - propertiesEntries[context.path] = incoming - } else { - for ((key, value) in incoming) { - if (props.containsKey(key)) { - when (MergeStrategy.from(mergeStrategyFor(context.path))) { - MergeStrategy.Latest -> { - props[key] = value - } - MergeStrategy.Append -> { - props[key] = props.getProperty(key as String) + mergeSeparatorFor(context.path) + value - } - MergeStrategy.First -> Unit - MergeStrategy.Fail -> { - val conflictsForPath = conflicts.computeIfAbsent(context.path) { mutableMapOf() } - conflictsForPath.compute(key as String) { _, count -> (count ?: 1) + 1 } - } + val props = propertiesEntries.computeIfAbsent(context.path) { ReproducibleProperties() }.props + val mergeStrategy = MergeStrategy.from(mergeStrategyFor(context.path)) + val mergeSeparator = if (mergeStrategy == MergeStrategy.Append) mergeSeparatorFor(context.path) else "" + loadAndTransformKeys(context.inputStream) { key, value -> + if (props.containsKey(key)) { + when (mergeStrategy) { + MergeStrategy.Latest -> { + props[key] = value + } + MergeStrategy.Append -> { + props[key] = props[key] + mergeSeparator + value + } + MergeStrategy.First -> Unit + MergeStrategy.Fail -> { + val conflictsForPath = conflicts.computeIfAbsent(context.path) { mutableMapOf() } + conflictsForPath.compute(key) { _, count -> (count ?: 1) + 1 } } - } else { - props[key] = value } + } else { + props[key] = value } } } - private fun loadAndTransformKeys(inputStream: InputStream): CleanProperties { - val props = CleanProperties() + private fun loadAndTransformKeys(inputStream: InputStream, keyValue: (key: String, value: String) -> Unit) { + val props = Properties() // InputStream closed by caller, so we don't do it here. props.load(inputStream.bufferedReader(charset)) - return transformKeys(props) - } - - private fun transformKeys(properties: Properties): CleanProperties { - if (keyTransformer == IDENTITY) { - return properties as CleanProperties - } - val result = CleanProperties() - properties.forEach { (key, value) -> - result[keyTransformer(key as String)] = value - } - return result + props.forEach { keyValue(keyTransformer(it.key as String), it.value as String) } } private fun mergeStrategyFor(path: String): String { @@ -234,13 +235,9 @@ public open class PropertiesFileTransformer @Inject constructor( } // Cannot close the writer as the OutputStream needs to remain open. - val zipWriter = os.writer(charset) propertiesEntries.forEach { (path, props) -> os.putNextEntry(zipEntry(path, preserveFileTimestamps)) - props.inputStream(charset).bufferedReader(charset).use { - it.copyTo(zipWriter) - } - zipWriter.flush() + props.writeProperties(charset, os, escapeUnicode.get()) os.closeEntry() } } diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt index ce8daec1f..7b492f9bf 100644 --- a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt @@ -74,7 +74,7 @@ class PropertiesFileTransformerTest : BaseTransformerTest Date: Tue, 18 Nov 2025 14:43:16 +0100 Subject: [PATCH 2/3] simplify --- api/shadow.api | 1 - .../PropertiesFileTransformerTest.kt | 30 ++++-- .../shadow/internal/ReproducibleProperties.kt | 79 +++++++-------- .../transformers/PropertiesFileTransformer.kt | 18 +--- .../internal/ReproduciblePropertiesTest.kt | 95 +++++++++++++++++++ .../PropertiesFileTransformerTest.kt | 10 +- 6 files changed, 160 insertions(+), 73 deletions(-) create mode 100644 src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproduciblePropertiesTest.kt diff --git a/api/shadow.api b/api/shadow.api index d633bb907..ecb12177c 100644 --- a/api/shadow.api +++ b/api/shadow.api @@ -439,7 +439,6 @@ public class com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesF public fun (Lorg/gradle/api/model/ObjectFactory;)V public fun canTransformResource (Lorg/gradle/api/file/FileTreeElement;)Z public fun getCharsetName ()Lorg/gradle/api/provider/Property; - public fun getEscapeUnicode ()Lorg/gradle/api/provider/Property; public fun getKeyTransformer ()Lkotlin/jvm/functions/Function1; public fun getMappings ()Lorg/gradle/api/provider/MapProperty; public fun getMergeSeparator ()Lorg/gradle/api/provider/Property; diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt index ac392e43d..386582c8a 100644 --- a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt @@ -51,13 +51,31 @@ class PropertiesFileTransformerTest : BaseTransformerTest() { runWithSuccess(shadowJarPath) val expected = when (strategy) { - MergeStrategy.First -> arrayOf("key1=one", "key2=one", "key3=two") - MergeStrategy.Latest -> arrayOf("key1=one", "key2=two", "key3=two") - MergeStrategy.Append -> arrayOf("key1=one", "key2=one;two", "key3=two") + MergeStrategy.First -> + """ + key1=one + key2=one + key3=two + + """ + MergeStrategy.Latest -> + """ + key1=one + key2=two + key3=two + + """ + MergeStrategy.Append -> + """ + key1=one + key2=one;two + key3=two + + """ else -> fail("Unexpected strategy: $strategy") - } - val content = outputShadowedJar.use { it.getContent("META-INF/test.properties") } - assertThat(content).contains(*expected) + }.trimIndent() + val content = outputShadowedJar.use { it.getContent("META-INF/test.properties") }.invariantEolString + assertThat(content).isEqualTo(expected) } } diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt index b918eee6d..bec3b154a 100644 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt @@ -1,60 +1,47 @@ package com.github.jengelman.gradle.plugins.shadow.internal import java.io.OutputStream +import java.io.StringWriter +import java.io.Writer import java.nio.charset.Charset -import java.util.HexFormat -import java.util.SortedMap +import java.util.Properties +import java.util.TreeMap /** - * Maintains a map of properties sorted by key, which can be written out to a text file with a consistent - * ordering to satisfy the requirements of reproducible builds. + * Provides functionality for reproducible serialization. */ -internal class ReproducibleProperties { - internal val props: SortedMap = sortedMapOf() +internal class ReproducibleProperties : Properties() { + // Just to prevent accidental misuse. + override fun store(writer: Writer?, comments: String?) { + throw UnsupportedOperationException("use writeWithoutComments()") + } - fun writeProperties(charset: Charset, os: OutputStream, escape: Boolean) { - val zipWriter = os.writer(charset) - props.forEach { (key, value) -> - zipWriter.write(convertString(key, isKey = true, escape)) - zipWriter.write("=") - zipWriter.write(convertString(value, isKey = false, escape)) - zipWriter.write("\n") - } - zipWriter.flush() + // Just to prevent accidental misuse. + override fun store(out: OutputStream?, comments: String?) { + throw UnsupportedOperationException("use writeWithoutComments()") } - private fun convertString( - str: String, - isKey: Boolean, - escape: Boolean, - ): String { - val len = str.length - val out = StringBuilder() - val hex = HexFormat.of().withUpperCase() - for (x in 0.. 61) && (aChar.code < 127)) { - out.append(if (aChar == '\\') "\\\\" else aChar) - continue - } - when (aChar) { - ' ' -> { - if (x == 0 || isKey) out.append('\\') - out.append(' ') - } - '\t' -> out.append("\\t") - '\n' -> out.append("\\n") - '\r' -> out.append("\\r") - '\u000c' -> out.append("\\f") - '=', ':', '#', '!' -> out.append('\\').append(aChar) - else -> if (escape && ((aChar.code < 0x0020) || (aChar.code > 0x007e))) { - out.append("\\u").append(hex.toHexDigits(aChar)) - } else { - out.append(aChar) + fun writeWithoutComments(charset: Charset, os: OutputStream) { + val bufferedReader = StringWriter().apply { + super.store(this, null) + }.toString().reader().buffered() + + os.bufferedWriter(charset).apply { + var line: String? = null + while (bufferedReader.readLine().also { line = it } != null) { + if (!line!!.startsWith("#")) { + write(line) + newLine() } } - } - return out.toString() + }.flush() } + + // yields the entries for Properties.store0() in sorted order + override val entries: MutableSet> + get() { + val sorted = TreeMap() + super.entries.forEach { sorted[it.key] = it.value } + return sorted.entries + } } diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt index d0b2e15da..ad191fd38 100644 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt @@ -122,21 +122,9 @@ public open class PropertiesFileTransformer @Inject constructor( @get:Input public open val mergeSeparator: Property = objectFactory.property(",") - /** - * Properties files are written without escaping Unicode characters using the character set - * configured by [charsetName]. - * - * Set this property to `true` to escape all Unicode characters in the properties file, producing - * ASCII compatible files. - */ - @get:Input - public open val escapeUnicode: Property = objectFactory.property(false) - /** * The character set to use when reading and writing property files. * Defaults to `ISO-8859-1`. - * - * See also [escapeUnicode]. */ @get:Input public open val charsetName: Property = objectFactory.property(Charsets.ISO_8859_1.name()) @@ -161,7 +149,7 @@ public open class PropertiesFileTransformer @Inject constructor( } override fun transform(context: TransformerContext) { - val props = propertiesEntries.computeIfAbsent(context.path) { ReproducibleProperties() }.props + val props = propertiesEntries.computeIfAbsent(context.path) { ReproducibleProperties() } val mergeStrategy = MergeStrategy.from(mergeStrategyFor(context.path)) val mergeSeparator = if (mergeStrategy == MergeStrategy.Append) mergeSeparatorFor(context.path) else "" loadAndTransformKeys(context.inputStream) { key, value -> @@ -171,7 +159,7 @@ public open class PropertiesFileTransformer @Inject constructor( props[key] = value } MergeStrategy.Append -> { - props[key] = props[key] + mergeSeparator + value + props[key] = props[key] as String + mergeSeparator + value } MergeStrategy.First -> Unit MergeStrategy.Fail -> { @@ -237,7 +225,7 @@ public open class PropertiesFileTransformer @Inject constructor( // Cannot close the writer as the OutputStream needs to remain open. propertiesEntries.forEach { (path, props) -> os.putNextEntry(zipEntry(path, preserveFileTimestamps)) - props.writeProperties(charset, os, escapeUnicode.get()) + props.writeWithoutComments(charset, os) os.closeEntry() } } diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproduciblePropertiesTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproduciblePropertiesTest.kt new file mode 100644 index 000000000..008f571b4 --- /dev/null +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproduciblePropertiesTest.kt @@ -0,0 +1,95 @@ +package com.github.jengelman.gradle.plugins.shadow.internal + +import assertk.assertThat +import assertk.assertions.isEqualTo +import java.io.ByteArrayOutputStream +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource + +class ReproduciblePropertiesTest { + @ParameterizedTest + @MethodSource("charsets") + fun emptyProperties(charset: Charset) { + val props = ReproducibleProperties() + + val str = props.writeToString(charset) + + assertThat(str).isEqualTo("") + } + + @ParameterizedTest + @MethodSource("charsets") + fun someProperties(charset: Charset) { + val props = ReproducibleProperties() + props["key"] = "value" + props["key2"] = "value2" + props["a"] = "b" + props["d"] = "e" + props["0"] = "1" + props["b"] = "c" + props["c"] = "d" + props["e"] = "f" + + val str = props.writeToString(charset) + + assertThat(str).isEqualTo( + """ + 0=1 + a=b + b=c + c=d + d=e + e=f + key=value + key2=value2 + + """.trimIndent(), + ) + } + + @ParameterizedTest + @MethodSource("charsetsUtf") + fun utfChars(charset: Charset) { + val props = ReproducibleProperties() + props["äöüß"] = "aouss" + props["áèô"] = "aeo" + props["€²³"] = "x" + props["传傳磨宿说説"] = "b" + + val str = props.writeToString(charset) + + assertThat(str).isEqualTo( + """ + áèô=aeo + äöüß=aouss + €²³=x + 传傳磨宿说説=b + + """.trimIndent(), + ) + } + + internal fun ReproducibleProperties.writeToString(charset: Charset): String { + val buffer = ByteArrayOutputStream() + writeWithoutComments(charset, buffer) + return buffer.toString(charset.name()).replace(System.lineSeparator(), "\n") + } + + private companion object { + @JvmStatic + fun charsets() = listOf( + StandardCharsets.ISO_8859_1, + StandardCharsets.UTF_8, + StandardCharsets.US_ASCII, + StandardCharsets.UTF_16, + ) + + @JvmStatic + fun charsetsUtf() = listOf( + StandardCharsets.UTF_8, + StandardCharsets.UTF_16, + ) + } +} diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt index 7b492f9bf..ce8daec1f 100644 --- a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt @@ -74,7 +74,7 @@ class PropertiesFileTransformerTest : BaseTransformerTest Date: Wed, 19 Nov 2025 09:33:09 +0800 Subject: [PATCH 3/3] Cleanups --- .../shadow/FindResourceInClasspathTest.kt | 2 +- .../PropertiesFileTransformerTest.kt | 38 ++++---- .../gradle/plugins/shadow/util/Paths.kt | 7 -- ...ucibleProperties.kt => CleanProperties.kt} | 21 ++-- .../transformers/PropertiesFileTransformer.kt | 21 ++-- .../shadow/internal/CleanPropertiesTest.kt | 89 +++++++++++++++++ .../internal/ReproduciblePropertiesTest.kt | 95 ------------------- .../gradle/plugins/shadow/testkit/Strings.kt | 9 ++ 8 files changed, 133 insertions(+), 149 deletions(-) rename src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/{ReproducibleProperties.kt => CleanProperties.kt} (62%) create mode 100644 src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanPropertiesTest.kt delete mode 100644 src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproduciblePropertiesTest.kt create mode 100644 src/testKit/kotlin/com/github/jengelman/gradle/plugins/shadow/testkit/Strings.kt diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/FindResourceInClasspathTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/FindResourceInClasspathTest.kt index 888542af7..115d940c1 100644 --- a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/FindResourceInClasspathTest.kt +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/FindResourceInClasspathTest.kt @@ -5,7 +5,7 @@ import assertk.assertThat import assertk.assertions.contains import assertk.assertions.doesNotContain import com.github.jengelman.gradle.plugins.shadow.tasks.FindResourceInClasspath -import com.github.jengelman.gradle.plugins.shadow.util.variantSeparatorsPathString +import com.github.jengelman.gradle.plugins.shadow.testkit.variantSeparatorsPathString import kotlin.io.path.appendText import org.junit.jupiter.api.Test diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt index 386582c8a..d77911e7a 100644 --- a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformerTest.kt @@ -4,9 +4,9 @@ import assertk.assertThat import assertk.assertions.contains import assertk.assertions.isEqualTo import com.github.jengelman.gradle.plugins.shadow.testkit.getContent +import com.github.jengelman.gradle.plugins.shadow.testkit.invariantEolString import com.github.jengelman.gradle.plugins.shadow.transformers.PropertiesFileTransformer.MergeStrategy import com.github.jengelman.gradle.plugins.shadow.util.Issue -import com.github.jengelman.gradle.plugins.shadow.util.invariantEolString import kotlin.io.path.appendText import org.gradle.testkit.runner.TaskOutcome.FAILED import org.junit.jupiter.api.Assertions.fail @@ -53,29 +53,29 @@ class PropertiesFileTransformerTest : BaseTransformerTest() { val expected = when (strategy) { MergeStrategy.First -> """ - key1=one - key2=one - key3=two - - """ + |key1=one + |key2=one + |key3=two + | + """.trimMargin() MergeStrategy.Latest -> """ - key1=one - key2=two - key3=two - - """ + |key1=one + |key2=two + |key3=two + | + """.trimMargin() MergeStrategy.Append -> """ - key1=one - key2=one;two - key3=two - - """ + |key1=one + |key2=one;two + |key3=two + | + """.trimMargin() else -> fail("Unexpected strategy: $strategy") - }.trimIndent() - val content = outputShadowedJar.use { it.getContent("META-INF/test.properties") }.invariantEolString - assertThat(content).isEqualTo(expected) + } + val content = outputShadowedJar.use { it.getContent("META-INF/test.properties") } + assertThat(content.invariantEolString).isEqualTo(expected) } } diff --git a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/util/Paths.kt b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/util/Paths.kt index 6860d4c50..79a2d5e5a 100644 --- a/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/util/Paths.kt +++ b/src/functionalTest/kotlin/com/github/jengelman/gradle/plugins/shadow/util/Paths.kt @@ -1,14 +1,7 @@ package com.github.jengelman.gradle.plugins.shadow.util -import java.nio.file.FileSystems import java.nio.file.Path import kotlin.io.path.readText import kotlin.io.path.writeText fun Path.prependText(text: String) = writeText(text + readText()) - -val String.invariantEolString: String get() = replace(System.lineSeparator(), "\n") - -val String.variantSeparatorsPathString: String get() = replace("/", fileSystem.separator) - -private val fileSystem = FileSystems.getDefault() diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt similarity index 62% rename from src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt rename to src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt index bec3b154a..87ac366dc 100644 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproducibleProperties.kt +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanProperties.kt @@ -5,19 +5,16 @@ import java.io.StringWriter import java.io.Writer import java.nio.charset.Charset import java.util.Properties -import java.util.TreeMap /** * Provides functionality for reproducible serialization. */ -internal class ReproducibleProperties : Properties() { - // Just to prevent accidental misuse. - override fun store(writer: Writer?, comments: String?) { +internal class CleanProperties : Properties() { + override fun store(writer: Writer, comments: String) { throw UnsupportedOperationException("use writeWithoutComments()") } - // Just to prevent accidental misuse. - override fun store(out: OutputStream?, comments: String?) { + override fun store(out: OutputStream, comments: String?) { throw UnsupportedOperationException("use writeWithoutComments()") } @@ -28,8 +25,8 @@ internal class ReproducibleProperties : Properties() { os.bufferedWriter(charset).apply { var line: String? = null - while (bufferedReader.readLine().also { line = it } != null) { - if (!line!!.startsWith("#")) { + while (bufferedReader.readLine().also { line = it } != null && line != null) { + if (!line.startsWith("#")) { write(line) newLine() } @@ -37,11 +34,7 @@ internal class ReproducibleProperties : Properties() { }.flush() } - // yields the entries for Properties.store0() in sorted order override val entries: MutableSet> - get() { - val sorted = TreeMap() - super.entries.forEach { sorted[it.key] = it.value } - return sorted.entries - } + // Yields the entries for Properties.store0() in sorted order. + get() = super.entries.toSortedSet(compareBy { it.key.toString() }) } diff --git a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt index ad191fd38..45d7aab58 100644 --- a/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt +++ b/src/main/kotlin/com/github/jengelman/gradle/plugins/shadow/transformers/PropertiesFileTransformer.kt @@ -1,6 +1,6 @@ package com.github.jengelman.gradle.plugins.shadow.transformers -import com.github.jengelman.gradle.plugins.shadow.internal.ReproducibleProperties +import com.github.jengelman.gradle.plugins.shadow.internal.CleanProperties import com.github.jengelman.gradle.plugins.shadow.internal.mapProperty import com.github.jengelman.gradle.plugins.shadow.internal.property import com.github.jengelman.gradle.plugins.shadow.internal.setProperty @@ -108,7 +108,7 @@ public open class PropertiesFileTransformer @Inject constructor( internal val conflicts: MutableMap> = mutableMapOf() @get:Internal - internal val propertiesEntries = mutableMapOf() + internal val propertiesEntries = mutableMapOf() @get:Input public open val paths: SetProperty = objectFactory.setProperty() @@ -149,17 +149,15 @@ public open class PropertiesFileTransformer @Inject constructor( } override fun transform(context: TransformerContext) { - val props = propertiesEntries.computeIfAbsent(context.path) { ReproducibleProperties() } - val mergeStrategy = MergeStrategy.from(mergeStrategyFor(context.path)) - val mergeSeparator = if (mergeStrategy == MergeStrategy.Append) mergeSeparatorFor(context.path) else "" + val props = propertiesEntries.computeIfAbsent(context.path) { CleanProperties() } loadAndTransformKeys(context.inputStream) { key, value -> if (props.containsKey(key)) { - when (mergeStrategy) { + when (MergeStrategy.from(mergeStrategyFor(context.path))) { MergeStrategy.Latest -> { props[key] = value } MergeStrategy.Append -> { - props[key] = props[key] as String + mergeSeparator + value + props[key] = props[key] as String + mergeSeparatorFor(context.path) + value } MergeStrategy.First -> Unit MergeStrategy.Fail -> { @@ -173,11 +171,9 @@ public open class PropertiesFileTransformer @Inject constructor( } } - private fun loadAndTransformKeys(inputStream: InputStream, keyValue: (key: String, value: String) -> Unit) { - val props = Properties() - // InputStream closed by caller, so we don't do it here. - props.load(inputStream.bufferedReader(charset)) - props.forEach { keyValue(keyTransformer(it.key as String), it.value as String) } + private fun loadAndTransformKeys(inputStream: InputStream, action: (key: String, value: String) -> Unit) { + val props = Properties().apply { load(inputStream.bufferedReader(charset)) } + props.forEach { action(keyTransformer(it.key as String), it.value as String) } } private fun mergeStrategyFor(path: String): String { @@ -222,7 +218,6 @@ public open class PropertiesFileTransformer @Inject constructor( error(message) } - // Cannot close the writer as the OutputStream needs to remain open. propertiesEntries.forEach { (path, props) -> os.putNextEntry(zipEntry(path, preserveFileTimestamps)) props.writeWithoutComments(charset, os) diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanPropertiesTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanPropertiesTest.kt new file mode 100644 index 000000000..be5c5bd58 --- /dev/null +++ b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/CleanPropertiesTest.kt @@ -0,0 +1,89 @@ +package com.github.jengelman.gradle.plugins.shadow.internal + +import assertk.assertThat +import assertk.assertions.isEqualTo +import com.github.jengelman.gradle.plugins.shadow.testkit.invariantEolString +import java.io.ByteArrayOutputStream +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource + +class CleanPropertiesTest { + @ParameterizedTest + @MethodSource("generalCharsetsProvider") + fun emptyProperties(charset: Charset) { + val output = CleanProperties().writeToString(charset) + + assertThat(output).isEqualTo("") + } + + @ParameterizedTest + @MethodSource("generalCharsetsProvider") + fun asciiProps(charset: Charset) { + val output = CleanProperties().also { props -> + props["key"] = "value" + props["key2"] = "value2" + props["a"] = "b" + props["d"] = "e" + props["0"] = "1" + props["b"] = "c" + props["c"] = "d" + props["e"] = "f" + }.writeToString(charset) + + assertThat(output).isEqualTo( + """ + |0=1 + |a=b + |b=c + |c=d + |d=e + |e=f + |key=value + |key2=value2 + | + """.trimMargin(), + ) + } + + @ParameterizedTest + @MethodSource("utfCharsetsProvider") + fun utfProps(charset: Charset) { + val output = CleanProperties().also { props -> + props["äöüß"] = "aouss" + props["áèô"] = "aeo" + props["€²³"] = "x" + props["传傳磨宿说説"] = "b" + }.writeToString(charset) + + assertThat(output).isEqualTo( + """ + |áèô=aeo + |äöüß=aouss + |€²³=x + |传傳磨宿说説=b + | + """.trimMargin(), + ) + } + + private companion object Companion { + @JvmStatic + fun generalCharsetsProvider() = listOf( + StandardCharsets.ISO_8859_1, + StandardCharsets.US_ASCII, + ) + utfCharsetsProvider() + + @JvmStatic + fun utfCharsetsProvider() = listOf( + StandardCharsets.UTF_8, + StandardCharsets.UTF_16, + ) + + fun CleanProperties.writeToString(charset: Charset): String { + return ByteArrayOutputStream().also { writeWithoutComments(charset, it) } + .toString(charset.name()).invariantEolString + } + } +} diff --git a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproduciblePropertiesTest.kt b/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproduciblePropertiesTest.kt deleted file mode 100644 index 008f571b4..000000000 --- a/src/test/kotlin/com/github/jengelman/gradle/plugins/shadow/internal/ReproduciblePropertiesTest.kt +++ /dev/null @@ -1,95 +0,0 @@ -package com.github.jengelman.gradle.plugins.shadow.internal - -import assertk.assertThat -import assertk.assertions.isEqualTo -import java.io.ByteArrayOutputStream -import java.nio.charset.Charset -import java.nio.charset.StandardCharsets -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.MethodSource - -class ReproduciblePropertiesTest { - @ParameterizedTest - @MethodSource("charsets") - fun emptyProperties(charset: Charset) { - val props = ReproducibleProperties() - - val str = props.writeToString(charset) - - assertThat(str).isEqualTo("") - } - - @ParameterizedTest - @MethodSource("charsets") - fun someProperties(charset: Charset) { - val props = ReproducibleProperties() - props["key"] = "value" - props["key2"] = "value2" - props["a"] = "b" - props["d"] = "e" - props["0"] = "1" - props["b"] = "c" - props["c"] = "d" - props["e"] = "f" - - val str = props.writeToString(charset) - - assertThat(str).isEqualTo( - """ - 0=1 - a=b - b=c - c=d - d=e - e=f - key=value - key2=value2 - - """.trimIndent(), - ) - } - - @ParameterizedTest - @MethodSource("charsetsUtf") - fun utfChars(charset: Charset) { - val props = ReproducibleProperties() - props["äöüß"] = "aouss" - props["áèô"] = "aeo" - props["€²³"] = "x" - props["传傳磨宿说説"] = "b" - - val str = props.writeToString(charset) - - assertThat(str).isEqualTo( - """ - áèô=aeo - äöüß=aouss - €²³=x - 传傳磨宿说説=b - - """.trimIndent(), - ) - } - - internal fun ReproducibleProperties.writeToString(charset: Charset): String { - val buffer = ByteArrayOutputStream() - writeWithoutComments(charset, buffer) - return buffer.toString(charset.name()).replace(System.lineSeparator(), "\n") - } - - private companion object { - @JvmStatic - fun charsets() = listOf( - StandardCharsets.ISO_8859_1, - StandardCharsets.UTF_8, - StandardCharsets.US_ASCII, - StandardCharsets.UTF_16, - ) - - @JvmStatic - fun charsetsUtf() = listOf( - StandardCharsets.UTF_8, - StandardCharsets.UTF_16, - ) - } -} diff --git a/src/testKit/kotlin/com/github/jengelman/gradle/plugins/shadow/testkit/Strings.kt b/src/testKit/kotlin/com/github/jengelman/gradle/plugins/shadow/testkit/Strings.kt new file mode 100644 index 000000000..8dac7b1dd --- /dev/null +++ b/src/testKit/kotlin/com/github/jengelman/gradle/plugins/shadow/testkit/Strings.kt @@ -0,0 +1,9 @@ +package com.github.jengelman.gradle.plugins.shadow.testkit + +import java.nio.file.FileSystems + +val String.invariantEolString: String get() = replace(System.lineSeparator(), "\n") + +val String.variantSeparatorsPathString: String get() = replace("/", fileSystem.separator) + +private val fileSystem = FileSystems.getDefault()