From 8d6e3d56fc2497d39724d74c8ce434771d8c84b5 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 22 May 2024 11:10:57 +0200 Subject: [PATCH 1/5] HDDS-10897. Refactor OzoneQuota --- .../apache/hadoop/hdds/client/OzoneQuota.java | 37 ++++++------------- .../apache/hadoop/hdds/client/QuotaList.java | 21 +++++++++-- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java index b3a762e2eda1..8a161a5e3734 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java @@ -20,11 +20,6 @@ import com.google.common.base.Strings; -import static org.apache.hadoop.ozone.OzoneConsts.GB; -import static org.apache.hadoop.ozone.OzoneConsts.KB; -import static org.apache.hadoop.ozone.OzoneConsts.MB; -import static org.apache.hadoop.ozone.OzoneConsts.TB; - /** * represents an OzoneQuota Object that can be applied to @@ -44,27 +39,17 @@ public enum Units { B, KB, MB, GB, TB } // Quota to decide how many buckets can be created. private long quotaInNamespace; // Quota to decide how many storage space will be used in bytes. - private long quotaInBytes; - private RawQuotaInBytes rawQuotaInBytes; + private final long quotaInBytes; + private final RawQuotaInBytes rawQuotaInBytes; // Data class of Quota. - private static QuotaList quotaList; - - /** Setting QuotaList parameters from large to small. */ - static { - quotaList = new QuotaList(); - quotaList.addQuotaList(OZONE_QUOTA_TB, Units.TB, TB); - quotaList.addQuotaList(OZONE_QUOTA_GB, Units.GB, GB); - quotaList.addQuotaList(OZONE_QUOTA_MB, Units.MB, MB); - quotaList.addQuotaList(OZONE_QUOTA_KB, Units.KB, KB); - quotaList.addQuotaList(OZONE_QUOTA_B, Units.B, 1L); - } + private static final QuotaList QUOTA_LIST = new QuotaList(); /** * Used to convert user input values into bytes such as: 1MB-> 1048576. */ private static class RawQuotaInBytes { - private Units unit; - private long size; + private final Units unit; + private final long size; RawQuotaInBytes(Units unit, long size) { this.unit = unit; @@ -84,9 +69,9 @@ public long getSize() { */ public long sizeInBytes() { long sQuota = -1L; - for (Units quota : quotaList.getUnitQuotaArray()) { + for (Units quota : QUOTA_LIST.getUnitQuotaArray()) { if (quota == this.unit) { - sQuota = quotaList.getQuotaSize(quota); + sQuota = QUOTA_LIST.getQuotaSize(quota); break; } } @@ -162,11 +147,11 @@ public static OzoneQuota parseSpaceQuota(String quotaInBytes) { Units currUnit = Units.B; try { - for (String quota : quotaList.getOzoneQuotaArray()) { + for (String quota : QUOTA_LIST.getOzoneQuotaArray()) { if (uppercase.endsWith((quota))) { size = uppercase .substring(0, uppercase.length() - quota.length()); - currUnit = quotaList.getUnits(quota); + currUnit = QUOTA_LIST.getUnits(quota); break; } } @@ -242,10 +227,10 @@ public static OzoneQuota getOzoneQuota(long quotaInBytes, long quotaInNamespace) { long size = 1L; Units unit = Units.B; - for (Long quota : quotaList.getSizeQuotaArray()) { + for (Long quota : QUOTA_LIST.getSizeQuotaArray()) { if (quotaInBytes % quota == 0) { size = quotaInBytes / quota; - unit = quotaList.getQuotaUnit(quota); + unit = QUOTA_LIST.getQuotaUnit(quota); } } return new OzoneQuota(quotaInNamespace, new RawQuotaInBytes(unit, size)); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java index 230b825f4d45..c00cf387f58d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java @@ -19,8 +19,14 @@ package org.apache.hadoop.hdds.client; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import static org.apache.hadoop.ozone.OzoneConsts.GB; +import static org.apache.hadoop.ozone.OzoneConsts.KB; +import static org.apache.hadoop.ozone.OzoneConsts.MB; +import static org.apache.hadoop.ozone.OzoneConsts.TB; + /** *This class contains arraylist for storage constant used in OzoneQuota. */ @@ -33,9 +39,16 @@ public QuotaList() { ozoneQuota = new ArrayList<>(); unitQuota = new ArrayList<>(); sizeQuota = new ArrayList<>(); + + // Setting QuotaList parameters from large to small. + addQuotaList(OzoneQuota.OZONE_QUOTA_TB, OzoneQuota.Units.TB, TB); + addQuotaList(OzoneQuota.OZONE_QUOTA_GB, OzoneQuota.Units.GB, GB); + addQuotaList(OzoneQuota.OZONE_QUOTA_MB, OzoneQuota.Units.MB, MB); + addQuotaList(OzoneQuota.OZONE_QUOTA_KB, OzoneQuota.Units.KB, KB); + addQuotaList(OzoneQuota.OZONE_QUOTA_B, OzoneQuota.Units.B, 1L); } - public void addQuotaList( + private void addQuotaList( String oQuota, OzoneQuota.Units uQuota, Long sQuota) { ozoneQuota.add(oQuota); unitQuota.add(uQuota); @@ -43,15 +56,15 @@ public void addQuotaList( } public List getOzoneQuotaArray() { - return this.ozoneQuota; + return Collections.unmodifiableList(this.ozoneQuota); } public List getSizeQuotaArray() { - return this.sizeQuota; + return Collections.unmodifiableList(this.sizeQuota); } public List getUnitQuotaArray() { - return this.unitQuota; + return Collections.unmodifiableList(this.unitQuota); } public OzoneQuota.Units getUnits(String oQuota) { From 7e5004b4e2593dd4950cd6e2800cf4ca9da6b2fc Mon Sep 17 00:00:00 2001 From: Tsz-Wo Nicholas Sze Date: Wed, 22 May 2024 11:12:32 +0200 Subject: [PATCH 2/5] remove QuotaList --- .../apache/hadoop/hdds/client/OzoneQuota.java | 85 ++++++++++++------- .../apache/hadoop/hdds/client/QuotaList.java | 82 ------------------ 2 files changed, 54 insertions(+), 113 deletions(-) delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java index 8a161a5e3734..bc0dc2d58265 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,7 +19,12 @@ package org.apache.hadoop.hdds.client; import com.google.common.base.Strings; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.ratis.util.Preconditions; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; /** * represents an OzoneQuota Object that can be applied to @@ -27,27 +32,59 @@ */ public final class OzoneQuota { - public static final String OZONE_QUOTA_B = "B"; - public static final String OZONE_QUOTA_KB = "KB"; - public static final String OZONE_QUOTA_MB = "MB"; - public static final String OZONE_QUOTA_GB = "GB"; - public static final String OZONE_QUOTA_TB = "TB"; - /** Quota Units.*/ - public enum Units { B, KB, MB, GB, TB } + public enum Units { + // the names and the ordering are important + B(1), + KB(OzoneConsts.KB), + MB(OzoneConsts.MB), + GB(OzoneConsts.GB), + TB(OzoneConsts.TB); + + private final long size; + private final List cache; + + Units(long size) { + this.size = size; + this.cache = createCache(this); + } + + private static List createCache(Units unit) { + final List quotas = new ArrayList<>(1024); + for(int i = 0; i < 1024; i++) { + quotas.add(new RawQuotaInBytes(unit, i)); + } + return Collections.unmodifiableList(quotas); + } + + public long getSize() { + return size; + } + + RawQuotaInBytes getRawQuotaInBytes(long size) { + return size < cache.size()? cache.get(Math.toIntExact(size)) + : new RawQuotaInBytes(this, size); + } + } // Quota to decide how many buckets can be created. private long quotaInNamespace; // Quota to decide how many storage space will be used in bytes. private final long quotaInBytes; private final RawQuotaInBytes rawQuotaInBytes; - // Data class of Quota. - private static final QuotaList QUOTA_LIST = new QuotaList(); /** * Used to convert user input values into bytes such as: 1MB-> 1048576. */ private static class RawQuotaInBytes { + static RawQuotaInBytes valueOf(long quotaInBytes) { + final int i = Long.numberOfTrailingZeros(quotaInBytes) / 10; + final Units unit = Units.values()[i]; + final RawQuotaInBytes b = unit.getRawQuotaInBytes(quotaInBytes >> (i *10)); + Preconditions.assertSame(quotaInBytes, b.sizeInBytes(), "sizeInBytes"); + return b; + } + private final Units unit; private final long size; @@ -68,14 +105,7 @@ public long getSize() { * Returns size in Bytes or negative num if there is no Quota. */ public long sizeInBytes() { - long sQuota = -1L; - for (Units quota : QUOTA_LIST.getUnitQuotaArray()) { - if (quota == this.unit) { - sQuota = QUOTA_LIST.getQuotaSize(quota); - break; - } - } - return this.getSize() * sQuota; + return this.getSize() * getUnit().getSize(); } @Override @@ -143,20 +173,21 @@ public static OzoneQuota parseSpaceQuota(String quotaInBytes) { String uppercase = quotaInBytes.toUpperCase() .replaceAll("\\s+", ""); String size = ""; - long nSize = 0; + final long nSize; Units currUnit = Units.B; try { - for (String quota : QUOTA_LIST.getOzoneQuotaArray()) { + for (Units unit : Units.values()) { + final String quota = unit.name(); if (uppercase.endsWith((quota))) { size = uppercase .substring(0, uppercase.length() - quota.length()); - currUnit = QUOTA_LIST.getUnits(quota); + currUnit = unit; break; } } // there might be no unit specified. - if (size.equals("")) { + if (size.isEmpty()) { size = uppercase; } nSize = Long.parseLong(size); @@ -225,15 +256,7 @@ public static OzoneQuota parseQuota(String quotaInBytes, */ public static OzoneQuota getOzoneQuota(long quotaInBytes, long quotaInNamespace) { - long size = 1L; - Units unit = Units.B; - for (Long quota : QUOTA_LIST.getSizeQuotaArray()) { - if (quotaInBytes % quota == 0) { - size = quotaInBytes / quota; - unit = QUOTA_LIST.getQuotaUnit(quota); - } - } - return new OzoneQuota(quotaInNamespace, new RawQuotaInBytes(unit, size)); + return new OzoneQuota(quotaInNamespace, RawQuotaInBytes.valueOf(quotaInBytes)); } public long getQuotaInNamespace() { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java deleted file mode 100644 index c00cf387f58d..000000000000 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/QuotaList.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.client; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.apache.hadoop.ozone.OzoneConsts.GB; -import static org.apache.hadoop.ozone.OzoneConsts.KB; -import static org.apache.hadoop.ozone.OzoneConsts.MB; -import static org.apache.hadoop.ozone.OzoneConsts.TB; - -/** - *This class contains arraylist for storage constant used in OzoneQuota. - */ -public class QuotaList { - private final ArrayList ozoneQuota; - private final ArrayList unitQuota; - private final ArrayList sizeQuota; - - public QuotaList() { - ozoneQuota = new ArrayList<>(); - unitQuota = new ArrayList<>(); - sizeQuota = new ArrayList<>(); - - // Setting QuotaList parameters from large to small. - addQuotaList(OzoneQuota.OZONE_QUOTA_TB, OzoneQuota.Units.TB, TB); - addQuotaList(OzoneQuota.OZONE_QUOTA_GB, OzoneQuota.Units.GB, GB); - addQuotaList(OzoneQuota.OZONE_QUOTA_MB, OzoneQuota.Units.MB, MB); - addQuotaList(OzoneQuota.OZONE_QUOTA_KB, OzoneQuota.Units.KB, KB); - addQuotaList(OzoneQuota.OZONE_QUOTA_B, OzoneQuota.Units.B, 1L); - } - - private void addQuotaList( - String oQuota, OzoneQuota.Units uQuota, Long sQuota) { - ozoneQuota.add(oQuota); - unitQuota.add(uQuota); - sizeQuota.add(sQuota); - } - - public List getOzoneQuotaArray() { - return Collections.unmodifiableList(this.ozoneQuota); - } - - public List getSizeQuotaArray() { - return Collections.unmodifiableList(this.sizeQuota); - } - - public List getUnitQuotaArray() { - return Collections.unmodifiableList(this.unitQuota); - } - - public OzoneQuota.Units getUnits(String oQuota) { - return unitQuota.get(ozoneQuota.indexOf(oQuota)); - } - - public Long getQuotaSize(OzoneQuota.Units uQuota) { - return sizeQuota.get(unitQuota.indexOf(uQuota)); - } - - public OzoneQuota.Units getQuotaUnit(Long sQuota) { - return unitQuota.get(sizeQuota.indexOf(sQuota)); - } - -} From 887d713e6cd6cf14ca531aa7f478d060974f6ac2 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 22 May 2024 11:15:46 +0200 Subject: [PATCH 3/5] fix checkstyle --- .../java/org/apache/hadoop/hdds/client/OzoneQuota.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java index bc0dc2d58265..ec8a7626abda 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java @@ -51,7 +51,7 @@ public enum Units { private static List createCache(Units unit) { final List quotas = new ArrayList<>(1024); - for(int i = 0; i < 1024; i++) { + for (int i = 0; i < 1024; i++) { quotas.add(new RawQuotaInBytes(unit, i)); } return Collections.unmodifiableList(quotas); @@ -61,9 +61,9 @@ public long getSize() { return size; } - RawQuotaInBytes getRawQuotaInBytes(long size) { - return size < cache.size()? cache.get(Math.toIntExact(size)) - : new RawQuotaInBytes(this, size); + RawQuotaInBytes getRawQuotaInBytes(long b) { + return b < cache.size() ? cache.get(Math.toIntExact(b)) + : new RawQuotaInBytes(this, b); } } @@ -80,7 +80,7 @@ private static class RawQuotaInBytes { static RawQuotaInBytes valueOf(long quotaInBytes) { final int i = Long.numberOfTrailingZeros(quotaInBytes) / 10; final Units unit = Units.values()[i]; - final RawQuotaInBytes b = unit.getRawQuotaInBytes(quotaInBytes >> (i *10)); + final RawQuotaInBytes b = unit.getRawQuotaInBytes(quotaInBytes >> (i * 10)); Preconditions.assertSame(quotaInBytes, b.sizeInBytes(), "sizeInBytes"); return b; } From 001bb610f88d3f286434dab1eb06e92e2232251d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 22 May 2024 12:16:15 +0200 Subject: [PATCH 4/5] fix parse order --- .../java/org/apache/hadoop/hdds/client/OzoneQuota.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java index ec8a7626abda..5e11266f4c9a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java @@ -23,6 +23,7 @@ import org.apache.ratis.util.Preconditions; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -67,6 +68,13 @@ RawQuotaInBytes getRawQuotaInBytes(long b) { } } + private static final List PARSE_ORDER; + static { + List reversed = new ArrayList<>(Arrays.asList(Units.values())); + Collections.reverse(reversed); + PARSE_ORDER = Collections.unmodifiableList(reversed); + } + // Quota to decide how many buckets can be created. private long quotaInNamespace; // Quota to decide how many storage space will be used in bytes. @@ -177,7 +185,7 @@ public static OzoneQuota parseSpaceQuota(String quotaInBytes) { Units currUnit = Units.B; try { - for (Units unit : Units.values()) { + for (Units unit : PARSE_ORDER) { final String quota = unit.name(); if (uppercase.endsWith((quota))) { size = uppercase From 120b4f0acf93ed1c39eaf33f765bedd3dacbf010 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 22 May 2024 18:25:32 +0200 Subject: [PATCH 5/5] add PB, EB --- .../main/java/org/apache/hadoop/hdds/client/OzoneQuota.java | 5 ++++- .../src/main/java/org/apache/hadoop/ozone/OzoneConsts.java | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java index 5e11266f4c9a..c8cf4fdd42ba 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java @@ -40,7 +40,9 @@ public enum Units { KB(OzoneConsts.KB), MB(OzoneConsts.MB), GB(OzoneConsts.GB), - TB(OzoneConsts.TB); + TB(OzoneConsts.TB), + PB(OzoneConsts.PB), + EB(OzoneConsts.EB); private final long size; private final List cache; @@ -86,6 +88,7 @@ RawQuotaInBytes getRawQuotaInBytes(long b) { */ private static class RawQuotaInBytes { static RawQuotaInBytes valueOf(long quotaInBytes) { + Preconditions.assertTrue(quotaInBytes >= 0, () -> "quotaInBytes = " + quotaInBytes + " must be >= 0"); final int i = Long.numberOfTrailingZeros(quotaInBytes) / 10; final Units unit = Units.values()[i]; final RawQuotaInBytes b = unit.getRawQuotaInBytes(quotaInBytes >> (i * 10)); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index f3c08b252b1f..4e3f7e01cbea 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -119,6 +119,8 @@ public final class OzoneConsts { public static final long MB = KB * 1024L; public static final long GB = MB * 1024L; public static final long TB = GB * 1024L; + public static final long PB = TB * 1024L; + public static final long EB = PB * 1024L; /** * level DB names used by SCM and data nodes.