From c44f596cdd942ec73011581b708d752c54fa1ec8 Mon Sep 17 00:00:00 2001 From: Marcel Pokrandt Date: Tue, 23 Jul 2019 13:17:29 +0200 Subject: [PATCH 1/7] open for different alphabets --- .../java/org/apache/commons/codec/binary/Base64.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/commons/codec/binary/Base64.java b/src/main/java/org/apache/commons/codec/binary/Base64.java index f515b510fe..f8e5809ce4 100644 --- a/src/main/java/org/apache/commons/codec/binary/Base64.java +++ b/src/main/java/org/apache/commons/codec/binary/Base64.java @@ -271,6 +271,14 @@ public Base64(final int lineLength, final byte[] lineSeparator) { * @since 1.4 */ public Base64(final int lineLength, final byte[] lineSeparator, final boolean urlSafe) { + this(lineLength, lineSeparator, urlSafe ? URL_SAFE_ENCODE_TABLE : STANDARD_ENCODE_TABLE); + } + + public Base64(byte[] encodeTable) { + this(0, CHUNK_SEPARATOR, encodeTable); + } + + public Base64(final int lineLength, final byte[] lineSeparator, byte[] encodingTable) { super(BYTES_PER_UNENCODED_BLOCK, BYTES_PER_ENCODED_BLOCK, lineLength, lineSeparator == null ? 0 : lineSeparator.length); @@ -294,7 +302,7 @@ public Base64(final int lineLength, final byte[] lineSeparator, final boolean ur this.lineSeparator = null; } this.decodeSize = this.encodeSize - 1; - this.encodeTable = urlSafe ? URL_SAFE_ENCODE_TABLE : STANDARD_ENCODE_TABLE; + this.encodeTable = encodingTable; } /** From 50aa3704a8bc025d249714a050ee733d82d4586d Mon Sep 17 00:00:00 2001 From: Marcel Pokrandt Date: Tue, 23 Jul 2019 13:37:03 +0200 Subject: [PATCH 2/7] open for different alphabets - add javadoc --- .../apache/commons/codec/binary/Base64.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/commons/codec/binary/Base64.java b/src/main/java/org/apache/commons/codec/binary/Base64.java index f8e5809ce4..69d0e5ecf4 100644 --- a/src/main/java/org/apache/commons/codec/binary/Base64.java +++ b/src/main/java/org/apache/commons/codec/binary/Base64.java @@ -274,10 +274,28 @@ public Base64(final int lineLength, final byte[] lineSeparator, final boolean ur this(lineLength, lineSeparator, urlSafe ? URL_SAFE_ENCODE_TABLE : STANDARD_ENCODE_TABLE); } - public Base64(byte[] encodeTable) { - this(0, CHUNK_SEPARATOR, encodeTable); + /** + * Creates a Base64 codec used for decoding and encoding with non-standard encodingTable-table + * + * @param encodingTable + * The manual encodeTable - a byte array of 64 chars + */ + public Base64(byte[] encodingTable) { + this(0, CHUNK_SEPARATOR, encodingTable); } + /** + * Creates a Base64 codec used for decoding and encoding with non-standard encode-table and given lineLength and lineSeparator + * + * @param lineLength + * Each line of encoded data will be at most of the given length (rounded down to nearest multiple of + * 4). If lineLength <= 0, then the output will not be divided into lines (chunks). Ignored when + * decoding. + * @param lineSeparator + * Each line of encoded data will end with this sequence of bytes. + * @param encodingTable + * The manual encodeTable - a byte array of 64 chars + */ public Base64(final int lineLength, final byte[] lineSeparator, byte[] encodingTable) { super(BYTES_PER_UNENCODED_BLOCK, BYTES_PER_ENCODED_BLOCK, lineLength, From 669e6f4c3662bfd531776975412ab71c763150a8 Mon Sep 17 00:00:00 2001 From: Marcel Pokrandt Date: Tue, 23 Jul 2019 22:00:12 +0200 Subject: [PATCH 3/7] open for different alphabets - add test --- .../apache/commons/codec/binary/Base64.java | 20 ++++++------ .../commons/codec/binary/Base64Test.java | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/apache/commons/codec/binary/Base64.java b/src/main/java/org/apache/commons/codec/binary/Base64.java index 69d0e5ecf4..09a34e25d6 100644 --- a/src/main/java/org/apache/commons/codec/binary/Base64.java +++ b/src/main/java/org/apache/commons/codec/binary/Base64.java @@ -79,7 +79,7 @@ public class Base64 extends BaseNCodec { * Thanks to "commons" project in ws.apache.org for this code. * http://svn.apache.org/repos/asf/webservices/commons/trunk/modules/util/ */ - private static final byte[] STANDARD_ENCODE_TABLE = { + public static final byte[] STANDARD_ENCODE_TABLE = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', @@ -92,7 +92,7 @@ public class Base64 extends BaseNCodec { * changed to - and _ to make the encoded Base64 results more URL-SAFE. * This table is only used when the Base64's mode is set to URL-SAFE. */ - private static final byte[] URL_SAFE_ENCODE_TABLE = { + public static final byte[] URL_SAFE_ENCODE_TABLE = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', @@ -275,13 +275,13 @@ public Base64(final int lineLength, final byte[] lineSeparator, final boolean ur } /** - * Creates a Base64 codec used for decoding and encoding with non-standard encodingTable-table + * Creates a Base64 codec used for decoding and encoding with non-standard encodeTable-table * - * @param encodingTable + * @param encodeTable * The manual encodeTable - a byte array of 64 chars */ - public Base64(byte[] encodingTable) { - this(0, CHUNK_SEPARATOR, encodingTable); + public Base64(byte[] encodeTable) { + this(0, CHUNK_SEPARATOR, encodeTable); } /** @@ -293,10 +293,10 @@ public Base64(byte[] encodingTable) { * decoding. * @param lineSeparator * Each line of encoded data will end with this sequence of bytes. - * @param encodingTable + * @param encodeTable * The manual encodeTable - a byte array of 64 chars */ - public Base64(final int lineLength, final byte[] lineSeparator, byte[] encodingTable) { + public Base64(final int lineLength, final byte[] lineSeparator, byte[] encodeTable) { super(BYTES_PER_UNENCODED_BLOCK, BYTES_PER_ENCODED_BLOCK, lineLength, lineSeparator == null ? 0 : lineSeparator.length); @@ -320,7 +320,7 @@ public Base64(final int lineLength, final byte[] lineSeparator, byte[] encodingT this.lineSeparator = null; } this.decodeSize = this.encodeSize - 1; - this.encodeTable = encodingTable; + this.encodeTable = encodeTable; } /** @@ -339,7 +339,7 @@ public boolean isUrlSafe() { * the data to encode, and once with inAvail set to "-1" to alert encoder that EOF has been reached, to flush last * remaining bytes (if not multiple of 3). *

- *

Note: no padding is added when encoding using the URL-safe alphabet.

+ *

Note: no padding is added when encoding not using the default alphabet.

*

* Thanks to "commons" project in ws.apache.org for the bitwise operations, and general approach. * http://svn.apache.org/repos/asf/webservices/commons/trunk/modules/util/ diff --git a/src/test/java/org/apache/commons/codec/binary/Base64Test.java b/src/test/java/org/apache/commons/codec/binary/Base64Test.java index a9b98673df..d8c3b5af95 100644 --- a/src/test/java/org/apache/commons/codec/binary/Base64Test.java +++ b/src/test/java/org/apache/commons/codec/binary/Base64Test.java @@ -32,6 +32,7 @@ import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.EncoderException; import org.apache.commons.lang3.ArrayUtils; +import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; @@ -115,6 +116,36 @@ public void testBase64() { assertEquals("decode hello world", "Hello World", decodeString); } + @Test + public void testCustomEncodingAlphabet() { + // create a duplicate of STANDARD_ENCODE_TABLE and replace two chars with + // custom values not already present in table + byte[] encodeTable = Arrays.copyOf(Base64.STANDARD_ENCODE_TABLE, 64); + encodeTable[0] = '.'; + encodeTable[1] = '-'; + + // two instances: one with default table and one with adjusted encoding table + Base64 b64 = new Base64(); + Base64 b64customEncoding = new Base64(encodeTable); + + final String content = "Hello World - this "; + + byte[] encodedBytes = b64.encode(StringUtils.getBytesUtf8(content)); + String encodedContent = StringUtils.newStringUtf8(encodedBytes); + + byte[] encodedBytesCustom = b64customEncoding.encode(StringUtils.getBytesUtf8(content)); + String encodedContentCustom = StringUtils.newStringUtf8(encodedBytesCustom); + + Assert.assertTrue("testing precondition not met - ecodedContent should contain parts of modified table", + encodedContent.contains("A") || encodedContent.contains("B")); + + Assert.assertEquals("custom encoding mismatch to expected - " + encodedContentCustom, + encodedContent + .replaceAll("A", ".").replaceAll("B", "-") // replace alphabet adjustments + .replaceAll("=", "") // remove padding (not default alphabet) + , encodedContentCustom); + } + @Test public void testBase64AtBufferStart() { testBase64InBuffer(0, 100); From 31ece681fd306a1e37fa1c71e5d11b90bc784aaf Mon Sep 17 00:00:00 2001 From: Marcel Pokrandt Date: Tue, 23 Jul 2019 23:35:18 +0200 Subject: [PATCH 4/7] open for different alphabets - add unfixed failing test for decoder --- .../java/org/apache/commons/codec/binary/Base64Test.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/apache/commons/codec/binary/Base64Test.java b/src/test/java/org/apache/commons/codec/binary/Base64Test.java index d8c3b5af95..db7ede1b8a 100644 --- a/src/test/java/org/apache/commons/codec/binary/Base64Test.java +++ b/src/test/java/org/apache/commons/codec/binary/Base64Test.java @@ -128,7 +128,7 @@ public void testCustomEncodingAlphabet() { Base64 b64 = new Base64(); Base64 b64customEncoding = new Base64(encodeTable); - final String content = "Hello World - this "; + final String content = "! Hello World - this ยง$%"; byte[] encodedBytes = b64.encode(StringUtils.getBytesUtf8(content)); String encodedContent = StringUtils.newStringUtf8(encodedBytes); @@ -144,6 +144,13 @@ public void testCustomEncodingAlphabet() { .replaceAll("A", ".").replaceAll("B", "-") // replace alphabet adjustments .replaceAll("=", "") // remove padding (not default alphabet) , encodedContentCustom); + + + // try decode encoded content + final byte[] decode = b64customEncoding.decode(encodedBytesCustom); + final String decodeString = StringUtils.newStringUtf8(decode); + + Assert.assertEquals(content, decodeString); } @Test From 4ed1cf9858f96e7bcce90d0c1a88c58e47efc6b0 Mon Sep 17 00:00:00 2001 From: Marcel Pokrandt Date: Tue, 23 Jul 2019 23:54:30 +0200 Subject: [PATCH 5/7] open for different alphabets - add calculating of decode table for custom encode table --- .../apache/commons/codec/binary/Base64.java | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/apache/commons/codec/binary/Base64.java b/src/main/java/org/apache/commons/codec/binary/Base64.java index 09a34e25d6..c9eae17c2a 100644 --- a/src/main/java/org/apache/commons/codec/binary/Base64.java +++ b/src/main/java/org/apache/commons/codec/binary/Base64.java @@ -111,7 +111,7 @@ public class Base64 extends BaseNCodec { * Thanks to "commons" project in ws.apache.org for this code. * http://svn.apache.org/repos/asf/webservices/commons/trunk/modules/util/ */ - private static final byte[] DECODE_TABLE = { + private static final byte[] DEFAULT_DECODE_TABLE = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 00-0f -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 10-1f @@ -134,14 +134,18 @@ public class Base64 extends BaseNCodec { // some state be preserved between calls of encode() and decode(). /** - * Encode table to use: either STANDARD or URL_SAFE. Note: the DECODE_TABLE above remains static because it is able - * to decode both STANDARD and URL_SAFE streams, but the encodeTable must be a member variable so we can switch - * between the two modes. + * Encode table to use: either STANDARD or URL_SAFE or custom. + * Note: the DEFAULT_DECODE_TABLE above remains static for STANDARD and URL_SAFA + * because it is able to decode both STANDARD and URL_SAFE streams, + * but the encodeTable must be a member variable so we can switch + * between modes. */ private final byte[] encodeTable; - // Only one decode table currently; keep for consistency with Base32 code - private final byte[] decodeTable = DECODE_TABLE; + /** + * Decode table to use + */ + private final byte[] decodeTable; /** * Line separator for encoding. Not used when decoding. Only used if lineLength > 0. @@ -300,6 +304,15 @@ public Base64(final int lineLength, final byte[] lineSeparator, byte[] encodeTab super(BYTES_PER_UNENCODED_BLOCK, BYTES_PER_ENCODED_BLOCK, lineLength, lineSeparator == null ? 0 : lineSeparator.length); + + this.encodeTable = encodeTable; + + if (encodeTable == STANDARD_ENCODE_TABLE || encodeTable == URL_SAFE_ENCODE_TABLE) { + decodeTable = DEFAULT_DECODE_TABLE; + } else { + decodeTable = calculateDecodeTable(encodeTable); + } + // TODO could be simplified if there is no requirement to reject invalid line sep when length <=0 // @see test case Base64Test.testConstructors() if (lineSeparator != null) { @@ -320,7 +333,23 @@ public Base64(final int lineLength, final byte[] lineSeparator, byte[] encodeTab this.lineSeparator = null; } this.decodeSize = this.encodeSize - 1; - this.encodeTable = encodeTable; + } + + /** + * calculates a decode table for a given encode table + * + * @param encodeTable + * @return decodeTable + */ + private byte[] calculateDecodeTable(byte[] encodeTable) { + byte[] decodeTable = new byte[256]; + for (int i=0; i < 256; i++) { + decodeTable[i] = -1; + } + for (int i=0; i < encodeTable.length; i++) { + decodeTable[(int) encodeTable[i]] = (byte) i; + } + return decodeTable; } /** @@ -467,8 +496,8 @@ void decode(final byte[] in, int inPos, final int inAvail, final Context context context.eof = true; break; } - if (b >= 0 && b < DECODE_TABLE.length) { - final int result = DECODE_TABLE[b]; + if (b >= 0 && b < decodeTable.length) { + final int result = decodeTable[b]; if (result >= 0) { context.modulus = (context.modulus+1) % BYTES_PER_ENCODED_BLOCK; context.ibitWorkArea = (context.ibitWorkArea << BITS_PER_ENCODED_BYTE) + result; @@ -535,7 +564,7 @@ public static boolean isArrayByteBase64(final byte[] arrayOctet) { * @since 1.4 */ public static boolean isBase64(final byte octet) { - return octet == PAD_DEFAULT || (octet >= 0 && octet < DECODE_TABLE.length && DECODE_TABLE[octet] != -1); + return octet == PAD_DEFAULT || (octet >= 0 && octet < DEFAULT_DECODE_TABLE.length && DEFAULT_DECODE_TABLE[octet] != -1); } /** From 8b29c523306cd69a924c4002cc9d7ffa735410cd Mon Sep 17 00:00:00 2001 From: Marcel Pokrandt Date: Wed, 24 Jul 2019 00:07:58 +0200 Subject: [PATCH 6/7] open for different alphabets - encode tables private again --- .../org/apache/commons/codec/binary/Base64.java | 4 ++-- .../apache/commons/codec/binary/Base64Test.java | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/apache/commons/codec/binary/Base64.java b/src/main/java/org/apache/commons/codec/binary/Base64.java index c9eae17c2a..8723a06045 100644 --- a/src/main/java/org/apache/commons/codec/binary/Base64.java +++ b/src/main/java/org/apache/commons/codec/binary/Base64.java @@ -79,7 +79,7 @@ public class Base64 extends BaseNCodec { * Thanks to "commons" project in ws.apache.org for this code. * http://svn.apache.org/repos/asf/webservices/commons/trunk/modules/util/ */ - public static final byte[] STANDARD_ENCODE_TABLE = { + private static final byte[] STANDARD_ENCODE_TABLE = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', @@ -92,7 +92,7 @@ public class Base64 extends BaseNCodec { * changed to - and _ to make the encoded Base64 results more URL-SAFE. * This table is only used when the Base64's mode is set to URL-SAFE. */ - public static final byte[] URL_SAFE_ENCODE_TABLE = { + private static final byte[] URL_SAFE_ENCODE_TABLE = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', diff --git a/src/test/java/org/apache/commons/codec/binary/Base64Test.java b/src/test/java/org/apache/commons/codec/binary/Base64Test.java index db7ede1b8a..ca07cffb16 100644 --- a/src/test/java/org/apache/commons/codec/binary/Base64Test.java +++ b/src/test/java/org/apache/commons/codec/binary/Base64Test.java @@ -118,11 +118,16 @@ public void testBase64() { @Test public void testCustomEncodingAlphabet() { - // create a duplicate of STANDARD_ENCODE_TABLE and replace two chars with + // created a duplicate of STANDARD_ENCODE_TABLE and replaced two chars with // custom values not already present in table - byte[] encodeTable = Arrays.copyOf(Base64.STANDARD_ENCODE_TABLE, 64); - encodeTable[0] = '.'; - encodeTable[1] = '-'; + // A => . B => - + byte[] encodeTable = { + '.', '-', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' + }; // two instances: one with default table and one with adjusted encoding table Base64 b64 = new Base64(); @@ -137,7 +142,7 @@ public void testCustomEncodingAlphabet() { String encodedContentCustom = StringUtils.newStringUtf8(encodedBytesCustom); Assert.assertTrue("testing precondition not met - ecodedContent should contain parts of modified table", - encodedContent.contains("A") || encodedContent.contains("B")); + encodedContent.contains("A") && encodedContent.contains("B")); Assert.assertEquals("custom encoding mismatch to expected - " + encodedContentCustom, encodedContent From 3e0d13c035021da7921cae9702194bb0699f5071 Mon Sep 17 00:00:00 2001 From: Marcel Pokrandt Date: Wed, 24 Jul 2019 00:15:33 +0200 Subject: [PATCH 7/7] open for different alphabets - check for encodeTable length --- src/main/java/org/apache/commons/codec/binary/Base64.java | 3 +++ .../java/org/apache/commons/codec/binary/Base64Test.java | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/src/main/java/org/apache/commons/codec/binary/Base64.java b/src/main/java/org/apache/commons/codec/binary/Base64.java index 8723a06045..c67c5bc5c0 100644 --- a/src/main/java/org/apache/commons/codec/binary/Base64.java +++ b/src/main/java/org/apache/commons/codec/binary/Base64.java @@ -310,6 +310,9 @@ public Base64(final int lineLength, final byte[] lineSeparator, byte[] encodeTab if (encodeTable == STANDARD_ENCODE_TABLE || encodeTable == URL_SAFE_ENCODE_TABLE) { decodeTable = DEFAULT_DECODE_TABLE; } else { + if (encodeTable.length != 64) { + throw new IllegalArgumentException("encodeTable must be exactly 64 bytes long"); + } decodeTable = calculateDecodeTable(encodeTable); } diff --git a/src/test/java/org/apache/commons/codec/binary/Base64Test.java b/src/test/java/org/apache/commons/codec/binary/Base64Test.java index ca07cffb16..7168409c16 100644 --- a/src/test/java/org/apache/commons/codec/binary/Base64Test.java +++ b/src/test/java/org/apache/commons/codec/binary/Base64Test.java @@ -116,6 +116,14 @@ public void testBase64() { assertEquals("decode hello world", "Hello World", decodeString); } + @Test(expected = IllegalArgumentException.class) + public void testCustomEncodingAlphabet_illegal() { + byte[] encodeTable = { + '.', '-', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M' + }; + Base64 b64customEncoding = new Base64(encodeTable); + } + @Test public void testCustomEncodingAlphabet() { // created a duplicate of STANDARD_ENCODE_TABLE and replaced two chars with