From 1fff619d0ed1d2759f8d2f219897e3f74df16661 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Thu, 22 Jul 2021 11:49:09 +0800 Subject: [PATCH] HDDS-5479. s3g bucket list failed when there is non-english key name. --- .../hadoop/ozone/s3/util/ContinueToken.java | 15 ++++++++------ .../ozone/s3/util/TestContinueToken.java | 20 +++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/ContinueToken.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/ContinueToken.java index 5d99f984e5f3..ce75c593476a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/ContinueToken.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/ContinueToken.java @@ -57,13 +57,16 @@ public ContinueToken(String lastKey, String lastDir) { public String encodeToString() { if (this.lastKey != null) { - ByteBuffer buffer = ByteBuffer - .allocate(4 + lastKey.length() - + (lastDir == null ? 0 : lastDir.length())); - buffer.putInt(lastKey.length()); - buffer.put(lastKey.getBytes(StandardCharsets.UTF_8)); + byte[] rawLastKey = lastKey.getBytes(StandardCharsets.UTF_8); + byte[] rawLastDir = (lastDir == null ? new byte[0] : + lastDir.getBytes(StandardCharsets.UTF_8)); + + ByteBuffer buffer = ByteBuffer.allocate( + 4 + rawLastKey.length + rawLastDir.length); + buffer.putInt(rawLastKey.length); + buffer.put(rawLastKey); if (lastDir != null) { - buffer.put(lastDir.getBytes(StandardCharsets.UTF_8)); + buffer.put(rawLastDir); } String hex = Hex.encodeHexString(buffer.array()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestContinueToken.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestContinueToken.java index a590367dfa03..05f9e7969477 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestContinueToken.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestContinueToken.java @@ -37,6 +37,26 @@ public void encodeDecode() throws OS3Exception { Assert.assertEquals(ct, parsedToken); } + @Test + public void encodeDecodeWithNonEnglishOne() throws OS3Exception { + ContinueToken ct = new ContinueToken("你好", null); + + ContinueToken parsedToken = + ContinueToken.decodeFromString(ct.encodeToString()); + + Assert.assertEquals(ct, parsedToken); + } + + @Test + public void encodeDecodeWithNonEnglishTwo() throws OS3Exception { + ContinueToken ct = new ContinueToken("你好", "上海"); + + ContinueToken parsedToken = + ContinueToken.decodeFromString(ct.encodeToString()); + + Assert.assertEquals(ct, parsedToken); + } + @Test public void encodeDecodeNullDir() throws OS3Exception { ContinueToken ct = new ContinueToken("key1", null);