From 9001603d736a4c4530183518ed5734c75059bf3d Mon Sep 17 00:00:00 2001 From: user <32928346+xichen01@users.noreply.github.com> Date: Thu, 3 Apr 2025 02:14:36 +0800 Subject: [PATCH 1/2] HDDS-12758. Reading and writing double dash prefixes will result in an error. --- .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../java/org/apache/hadoop/ozone/OmUtils.java | 20 ++++++++- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 31 ++++++++++++++ .../TestOzoneFSWithObjectStoreCreate.java | 42 +++++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index d03cc2a22fe5..11065a27ec34 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -170,6 +170,7 @@ public final class OzoneConsts { */ public static final String OM_KEY_PREFIX = "/"; + public static final String DOUBLE_SLASH_OM_KEY_PREFIX = "//"; public static final String OM_USER_PREFIX = "$"; public static final String OM_S3_PREFIX = "S3:"; public static final String OM_S3_CALLER_CONTEXT_PREFIX = "S3Auth:S3G|"; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 100ff74ed5e6..bef389102e20 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hdds.HddsUtils.getHostName; import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys; import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys; +import static org.apache.hadoop.ozone.OzoneConsts.DOUBLE_SLASH_OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; @@ -760,7 +761,7 @@ public static String normalizeKey(String keyName, if (!StringUtils.isBlank(keyName)) { String normalizedKeyName; if (keyName.startsWith(OM_KEY_PREFIX)) { - normalizedKeyName = new Path(keyName).toUri().getPath(); + normalizedKeyName = new Path(normalizeDoubleSlashPath(keyName)).toUri().getPath(); } else { normalizedKeyName = new Path(OM_KEY_PREFIX + keyName) .toUri().getPath(); @@ -778,6 +779,23 @@ public static String normalizeKey(String keyName, return keyName; } + /** + * Normalizes paths that start with double slashes to avoid URI authority parsing issues. + * This prevents Path parsing issues where paths starting with "//" have the content + * after "//" interpreted as the URI authority rather than as part of the path. + * For example: Path("//dir1").toUri().getAuthority() returns "dir1" and getPath() returns "" + */ + private static String normalizeDoubleSlashPath(String keyName) { + if (keyName.startsWith(DOUBLE_SLASH_OM_KEY_PREFIX)) { + int doubleSlashLen = DOUBLE_SLASH_OM_KEY_PREFIX.length(); + if (keyName.length() > doubleSlashLen && keyName.charAt(doubleSlashLen) != OM_KEY_PREFIX.charAt(0)) { + keyName = OM_KEY_PREFIX + keyName.substring(2); + } + return new Path(keyName).toUri().getPath(); + } + return keyName; + } + /** * Normalizes a given path up to the bucket level. * diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index a4eefb81dba7..e0cbc6b5e4d9 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -101,6 +101,9 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.s3.S3ClientFactory; import org.apache.hadoop.ozone.s3.S3GatewayService; import org.apache.ozone.test.OzoneTestBase; @@ -365,6 +368,34 @@ public void testPutObject() { assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); } + @Test + public void testPutDoubleSlashPrefixObject() { + final String bucketName = getBucketName(); + final String keyName = "//dir1"; + final String content = "bar"; + OzoneConfiguration conf = cluster.getConf(); + // Create a FSO bucket for test + try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { + ObjectStore store = ozoneClient.getObjectStore(); + OzoneVolume volume = store.getS3Volume(); + OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder() + .setVolumeName(volume.getName()) + .setBucketName(bucketName) + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); + OzoneManagerProtocol ozoneManagerProtocol = store.getClientProxy().getOzoneManagerClient(); + ozoneManagerProtocol.createBucket(bucketInfo.build()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); + + S3Object object = s3Client.getObject(bucketName, keyName); + assertEquals(content.length(), object.getObjectMetadata().getContentLength()); + } + @Test public void testPutObjectEmpty() { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index 0f5f24fd6609..2be94429fbd9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java @@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.FileNotFoundException; import java.io.IOException; @@ -386,6 +387,47 @@ public void testListKeysWithNotNormalizedPath() throws Exception { checkKeyList(ozoneKeyIterator, keys); } + @Test + public void testDoubleSlashPrefixPathNormalization() throws Exception { + OzoneVolume ozoneVolume = + client.getObjectStore().getVolume(volumeName); + OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); + String doubleSlashKey = "//dir1/key1"; + String normalizedDoubleSlashKey = "dir1/key1"; + String tripleSlashKey = "///dir2/key2"; + String normalizedTripleSlashKey = "dir2/key2"; + byte[] data = new byte[10]; + Arrays.fill(data, (byte)96); + ArrayList expectedKeys = new ArrayList<>(); + expectedKeys.add("dir1/"); + expectedKeys.add(normalizedDoubleSlashKey); + expectedKeys.add("dir2/"); + expectedKeys.add(normalizedTripleSlashKey); + + OzoneOutputStream stream1 = ozoneBucket.createKey(doubleSlashKey, data.length); + stream1.write(data, 0, data.length); + stream1.close(); + OzoneOutputStream stream2 = ozoneBucket.createKey(tripleSlashKey, data.length); + stream2.write(data, 0, data.length); + stream2.close(); + + try { + ozoneBucket.readKey(doubleSlashKey).close(); + ozoneBucket.readKey(normalizedDoubleSlashKey).close(); + } catch (Exception e) { + fail("Should be able to read key " + e.getMessage()); + } + try { + ozoneBucket.readKey(tripleSlashKey).close(); + ozoneBucket.readKey(normalizedTripleSlashKey).close(); + } catch (Exception e) { + fail("Should be able to read key " + e.getMessage()); + } + + Iterator it = ozoneBucket.listKeys("//dir"); + checkKeyList(it, expectedKeys); + } + private void checkKeyList(Iterator ozoneKeyIterator, List keys) { From ff38e37ce068b29f6c945212c94d1fde0817c2f8 Mon Sep 17 00:00:00 2001 From: user <32928346+xichen01@users.noreply.github.com> Date: Sat, 5 Apr 2025 01:40:10 +0800 Subject: [PATCH 2/2] Remove redundant slash and improve test cases --- .../java/org/apache/hadoop/ozone/OmUtils.java | 17 +++---- .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 4 +- .../TestOzoneFSWithObjectStoreCreate.java | 49 ++++++++----------- .../ozone/om/request/TestNormalizePaths.java | 3 ++ 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index bef389102e20..e71e5b912ac6 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -761,7 +761,7 @@ public static String normalizeKey(String keyName, if (!StringUtils.isBlank(keyName)) { String normalizedKeyName; if (keyName.startsWith(OM_KEY_PREFIX)) { - normalizedKeyName = new Path(normalizeDoubleSlashPath(keyName)).toUri().getPath(); + normalizedKeyName = new Path(normalizeLeadingSlashes(keyName)).toUri().getPath(); } else { normalizedKeyName = new Path(OM_KEY_PREFIX + keyName) .toUri().getPath(); @@ -780,18 +780,15 @@ public static String normalizeKey(String keyName, } /** - * Normalizes paths that start with double slashes to avoid URI authority parsing issues. - * This prevents Path parsing issues where paths starting with "//" have the content - * after "//" interpreted as the URI authority rather than as part of the path. - * For example: Path("//dir1").toUri().getAuthority() returns "dir1" and getPath() returns "" + * Normalizes paths by replacing multiple leading slashes with a single slash. */ - private static String normalizeDoubleSlashPath(String keyName) { + private static String normalizeLeadingSlashes(String keyName) { if (keyName.startsWith(DOUBLE_SLASH_OM_KEY_PREFIX)) { - int doubleSlashLen = DOUBLE_SLASH_OM_KEY_PREFIX.length(); - if (keyName.length() > doubleSlashLen && keyName.charAt(doubleSlashLen) != OM_KEY_PREFIX.charAt(0)) { - keyName = OM_KEY_PREFIX + keyName.substring(2); + int index = 0; + while (index < keyName.length() && keyName.charAt(index) == OM_KEY_PREFIX.charAt(0)) { + index++; } - return new Path(keyName).toUri().getPath(); + return OM_KEY_PREFIX + keyName.substring(index); } return keyName; } diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index e0cbc6b5e4d9..c45b37eed911 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -369,7 +369,7 @@ public void testPutObject() { } @Test - public void testPutDoubleSlashPrefixObject() { + public void testPutDoubleSlashPrefixObject() throws IOException { final String bucketName = getBucketName(); final String keyName = "//dir1"; final String content = "bar"; @@ -384,8 +384,6 @@ public void testPutDoubleSlashPrefixObject() { .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); OzoneManagerProtocol ozoneManagerProtocol = store.getClientProxy().getOzoneManagerClient(); ozoneManagerProtocol.createBucket(bucketInfo.build()); - } catch (IOException e) { - throw new RuntimeException(e); } InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index 2be94429fbd9..26f06f9abc7d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java @@ -68,6 +68,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Class tests create with object store and getFileStatus. @@ -387,44 +389,35 @@ public void testListKeysWithNotNormalizedPath() throws Exception { checkKeyList(ozoneKeyIterator, keys); } - @Test - public void testDoubleSlashPrefixPathNormalization() throws Exception { - OzoneVolume ozoneVolume = - client.getObjectStore().getVolume(volumeName); + @ParameterizedTest + @ValueSource(ints = {2, 3, 4}) + public void testDoubleSlashPrefixPathNormalization(int slashCount) throws Exception { + OzoneVolume ozoneVolume = client.getObjectStore().getVolume(volumeName); OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); - String doubleSlashKey = "//dir1/key1"; - String normalizedDoubleSlashKey = "dir1/key1"; - String tripleSlashKey = "///dir2/key2"; - String normalizedTripleSlashKey = "dir2/key2"; + // Generate a path with the specified number of leading slashes + StringBuilder keyPrefix = new StringBuilder(); + for (int i = 0; i < slashCount; i++) { + keyPrefix.append('/'); + } + String dirPath = "dir" + slashCount + "/"; + String keyName = "key" + slashCount; + String slashyKey = keyPrefix + dirPath + keyName; + String normalizedKey = dirPath + keyName; byte[] data = new byte[10]; Arrays.fill(data, (byte)96); ArrayList expectedKeys = new ArrayList<>(); - expectedKeys.add("dir1/"); - expectedKeys.add(normalizedDoubleSlashKey); - expectedKeys.add("dir2/"); - expectedKeys.add(normalizedTripleSlashKey); - - OzoneOutputStream stream1 = ozoneBucket.createKey(doubleSlashKey, data.length); - stream1.write(data, 0, data.length); - stream1.close(); - OzoneOutputStream stream2 = ozoneBucket.createKey(tripleSlashKey, data.length); - stream2.write(data, 0, data.length); - stream2.close(); + expectedKeys.add(dirPath); + expectedKeys.add(normalizedKey); + TestDataUtil.createKey(ozoneBucket, slashyKey, data); try { - ozoneBucket.readKey(doubleSlashKey).close(); - ozoneBucket.readKey(normalizedDoubleSlashKey).close(); - } catch (Exception e) { - fail("Should be able to read key " + e.getMessage()); - } - try { - ozoneBucket.readKey(tripleSlashKey).close(); - ozoneBucket.readKey(normalizedTripleSlashKey).close(); + ozoneBucket.readKey(slashyKey).close(); + ozoneBucket.readKey(normalizedKey).close(); } catch (Exception e) { fail("Should be able to read key " + e.getMessage()); } - Iterator it = ozoneBucket.listKeys("//dir"); + Iterator it = ozoneBucket.listKeys(dirPath); checkKeyList(it, expectedKeys); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java index c37cbfda5945..0889a532727f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java @@ -58,6 +58,9 @@ public void testNormalizePathsEnabled() throws Exception { validateAndNormalizeKey(true, "a/b/c/d/")); assertEquals("a/b/c/...../d", validateAndNormalizeKey(true, "////a/b/////c/...../d/")); + assertEquals("a/b/c", validateAndNormalizeKey(true, "/a/b/c")); + assertEquals("a/b/c", validateAndNormalizeKey(true, "//a/b/c")); + assertEquals("a/b/c", validateAndNormalizeKey(true, "///a/b/c")); } @Test