From 1dbb12c96d87f4160238f006c05b3502fedf5bd9 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 19 Feb 2025 16:04:39 +0800 Subject: [PATCH 1/3] Usse uuidv7 to gen multipart upload id so the same key would sort by time in listMultipartUpload response --- .../org/apache/hadoop/ozone/util/UUIDv7.java | 61 +++++++++++++++++++ .../s3/awssdk/v1/AbstractS3SDKV1Tests.java | 5 +- .../request/util/OMMultipartUploadUtils.java | 4 +- 3 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java new file mode 100644 index 000000000000..7074dfb3a0d9 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.util; + +import java.nio.ByteBuffer; +import java.security.SecureRandom; +import java.util.UUID; + +/** + * Utility class for generating UUIDv7. + * https://antonz.org/uuidv7/ + */ +public final class UUIDv7 { + private static final SecureRandom RANDOM = new SecureRandom(); + + // Private constructor to prevent instantiation + private UUIDv7() { + } + + public static UUID randomUUID() { + byte[] value = randomBytes(); + ByteBuffer buf = ByteBuffer.wrap(value); + long high = buf.getLong(); + long low = buf.getLong(); + return new UUID(high, low); + } + + public static byte[] randomBytes() { + // random bytes + byte[] value = new byte[16]; + RANDOM.nextBytes(value); + + // current timestamp in ms + ByteBuffer timestamp = ByteBuffer.allocate(Long.BYTES); + timestamp.putLong(System.currentTimeMillis()); + + // timestamp + System.arraycopy(timestamp.array(), 2, value, 0, 6); + + // version and variant + value[6] = (byte) ((value[6] & 0x0F) | 0x70); + value[8] = (byte) ((value[8] & 0x3F) | 0x80); + + return value; + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 9d9bc72eb235..17b62367db35 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -673,13 +673,10 @@ public void testListMultipartUploads() { uploadIds.add(uploadId1); String uploadId2 = initiateMultipartUpload(bucketName, multipartKey1, null, null, null); uploadIds.add(uploadId2); - // TODO: Currently, Ozone sorts based on uploadId instead of MPU init time within the same key. - // Remove this sorting step once HDDS-11532 has been implemented - Collections.sort(uploadIds); + String uploadId3 = initiateMultipartUpload(bucketName, multipartKey2, null, null, null); uploadIds.add(uploadId3); - // TODO: Add test for max uploads threshold and marker once HDDS-11530 has been implemented ListMultipartUploadsRequest listMultipartUploadsRequest = new ListMultipartUploadsRequest(bucketName); MultipartUploadListing result = s3Client.listMultipartUploads(listMultipartUploadsRequest); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java index 3c42d50b2bf7..9bee8b144036 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java @@ -20,12 +20,12 @@ import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import java.io.IOException; -import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.utils.UniqueId; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.util.UUIDv7; /** * Utility class related to OM Multipart Upload. @@ -41,7 +41,7 @@ private OMMultipartUploadUtils() { * @return multipart upload ID */ public static String getMultipartUploadId() { - return UUID.randomUUID() + "-" + UniqueId.next(); + return UUIDv7.randomUUID() + "-" + UniqueId.next(); } /** From eb8d6c7d8055245fcca562c855521ee8cff3da1e Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 19 Feb 2025 21:30:23 +0800 Subject: [PATCH 2/3] move uuidV7 to hdds module util --- .../common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {hadoop-ozone => hadoop-hdds}/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java (100%) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java similarity index 100% rename from hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java From 111d4deea762c9d305842a5a51f8c07715067e6d Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 19 Feb 2025 22:37:39 +0800 Subject: [PATCH 3/3] Make UUIDv7 random generator thread-local for better performance --- .../src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java index 7074dfb3a0d9..2a5f8f84b6db 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDv7.java @@ -26,7 +26,7 @@ * https://antonz.org/uuidv7/ */ public final class UUIDv7 { - private static final SecureRandom RANDOM = new SecureRandom(); + private static final ThreadLocal GENERATOR = ThreadLocal.withInitial(SecureRandom::new); // Private constructor to prevent instantiation private UUIDv7() { @@ -43,7 +43,7 @@ public static UUID randomUUID() { public static byte[] randomBytes() { // random bytes byte[] value = new byte[16]; - RANDOM.nextBytes(value); + GENERATOR.get().nextBytes(value); // current timestamp in ms ByteBuffer timestamp = ByteBuffer.allocate(Long.BYTES);