From 056297c08e352f67de5ba754e76cd4ba5883e682 Mon Sep 17 00:00:00 2001 From: Yuan Gu Date: Mon, 21 Jun 2021 21:36:22 -0400 Subject: [PATCH 01/15] set up basic file structure and renaming-related changes (cherry picked from commit 701c7c1abbbdb3ca3ecc5778229ad73a54d59af4) --- .../ozone/recon/ReconControllerModule.java | 4 +- .../ozone/recon/api/types/NSSummary.java | 48 +++++++++++++++++++ .../ozone/recon/codec/NSSummaryCodec.java | 24 ++++++++++ .../recon/spi/NSSummaryServiceProvider.java | 5 ++ .../impl/ContainerDBServiceProviderImpl.java | 2 +- .../impl/NSSummaryServiceProviderImpl.java | 9 ++++ .../recon/spi/impl/ReconDBDefinition.java | 17 +++++-- ...erDBProvider.java => ReconDBProvider.java} | 8 ++-- .../hadoop/ozone/recon/ReconTestInjector.java | 4 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 1 - ...Provider.java => TestReconDBProvider.java} | 10 ++-- 11 files changed, 112 insertions(+), 20 deletions(-) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/NSSummaryServiceProvider.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/NSSummaryServiceProviderImpl.java rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/{ReconContainerDBProvider.java => ReconDBProvider.java} (92%) rename hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/{TestReconContainerDBProvider.java => TestReconDBProvider.java} (88%) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index 8ddc8acd5dd0..81bbe4391b49 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -43,7 +43,7 @@ import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerDBProvider; +import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask; import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; @@ -86,7 +86,7 @@ protected void configure() { bind(OzoneConfiguration.class).toProvider(ConfigurationProvider.class); bind(ReconHttpServer.class).in(Singleton.class); bind(DBStore.class) - .toProvider(ReconContainerDBProvider.class).in(Singleton.class); + .toProvider(ReconDBProvider.class).in(Singleton.class); bind(ReconOMMetadataManager.class) .to(ReconOmMetadataManagerImpl.class); bind(OMMetadataManager.class).to(ReconOmMetadataManagerImpl.class); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java new file mode 100644 index 000000000000..aec86e27675e --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java @@ -0,0 +1,48 @@ +package org.apache.hadoop.ozone.recon.api.types; + +import java.util.Arrays; + +/** + * Class to encapsulate namespace metadata summaries from OM + */ + +public class NSSummary { + private int numOfFiles; + private int sizeOfFiles; + private int[] fileSizeBucket; + + public NSSummary() { + this.numOfFiles = 0; + this.sizeOfFiles = 0; + // < 1KB, 1KB - 512KB, 513KB - 1024KB, 1MB - 10MB, 10MB - 100MB, + // 100MB - 1000MB, 1GB - 10GB, 10GB - 100GB, 100GB - 1000GB, > 1TB + // TODO: can we generate a more reasonable bucket + // Should this file size distribution be fixed or dynamic based on + // the actual situation? + this.fileSizeBucket = new int[10]; + } + + public int getNumOfFiles() { + return numOfFiles; + } + + public int getSizeOfFiles() { + return sizeOfFiles; + } + + public int[] getFileSizeBucket() { + return fileSizeBucket; + } + + public void setNumOfFiles(int numOfFiles) { + this.numOfFiles = numOfFiles; + } + + public void setSizeOfFiles(int sizeOfFiles) { + this.sizeOfFiles = sizeOfFiles; + } + + public void setFileSizeBucket(int[] fileSizeBucket) { + this.fileSizeBucket = Arrays.copyOf(fileSizeBucket, 10); + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java new file mode 100644 index 000000000000..8e0c6aba81a5 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -0,0 +1,24 @@ +package org.apache.hadoop.ozone.recon.codec; + +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.hdds.utils.db.Codec; + +import java.io.IOException; + +public class NSSummaryCodec implements Codec{ + + @Override + public byte[] toPersistedFormat(NSSummary object) throws IOException { + return new byte[0]; + } + + @Override + public NSSummary fromPersistedFormat(byte[] rawData) throws IOException { + return null; + } + + @Override + public NSSummary copyObject(NSSummary object) { + return null; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/NSSummaryServiceProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/NSSummaryServiceProvider.java new file mode 100644 index 000000000000..78b7c68b699f --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/NSSummaryServiceProvider.java @@ -0,0 +1,5 @@ +package org.apache.hadoop.ozone.recon.spi; + +public interface NSSummaryServiceProvider { + +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java index f613558d50de..c3e4e50f38ba 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.recon.spi.impl; import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_COUNT_KEY; -import static org.apache.hadoop.ozone.recon.spi.impl.ReconContainerDBProvider.getNewDBStore; +import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider.getNewDBStore; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.CONTAINER_KEY; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.CONTAINER_KEY_COUNT; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.REPLICA_HISTORY; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/NSSummaryServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/NSSummaryServiceProviderImpl.java new file mode 100644 index 000000000000..6b259429ffb2 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/NSSummaryServiceProviderImpl.java @@ -0,0 +1,9 @@ +package org.apache.hadoop.ozone.recon.spi.impl; + +import com.google.inject.Provider; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.ozone.recon.spi.NSSummaryServiceProvider; + +public class NSSummaryServiceProviderImpl implements NSSummaryServiceProvider { + +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java index a9b23cea86d0..a3ce33214a5f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java @@ -18,14 +18,13 @@ */ package org.apache.hadoop.ozone.recon.spi.impl; -import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; -import org.apache.hadoop.hdds.utils.db.DBDefinition; -import org.apache.hadoop.hdds.utils.db.IntegerCodec; -import org.apache.hadoop.hdds.utils.db.LongCodec; +import org.apache.hadoop.hdds.utils.db.*; import org.apache.hadoop.ozone.recon.ReconServerConfigKeys; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; import org.apache.hadoop.ozone.recon.codec.ContainerReplicaHistoryListCodec; +import org.apache.hadoop.ozone.recon.codec.NSSummaryCodec; import org.apache.hadoop.ozone.recon.scm.ContainerReplicaHistoryList; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; /** * RocksDB definition for the DB internal to Recon. @@ -65,6 +64,14 @@ public ReconDBDefinition(String dbName) { ContainerReplicaHistoryList.class, new ContainerReplicaHistoryListCodec()); + public static final DBColumnFamilyDefinition + NAMESPACE_SUMMARY = new DBColumnFamilyDefinition( + "namespaceSummaryTable", + String.class, + new StringCodec(), + NSSummary.class, + new NSSummaryCodec()); + @Override public String getName() { return dbName; @@ -78,6 +85,6 @@ public String getLocationConfigKey() { @Override public DBColumnFamilyDefinition[] getColumnFamilies() { return new DBColumnFamilyDefinition[] { - CONTAINER_KEY, CONTAINER_KEY_COUNT, REPLICA_HISTORY}; + CONTAINER_KEY, CONTAINER_KEY_COUNT, REPLICA_HISTORY, NAMESPACE_SUMMARY}; } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerDBProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java similarity index 92% rename from hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerDBProvider.java rename to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java index ec36597a0816..9242d75f7111 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerDBProvider.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java @@ -38,18 +38,18 @@ /** * Provider for the Recon container DB (Metadata store). */ -public class ReconContainerDBProvider implements Provider { +public class ReconDBProvider implements Provider { @VisibleForTesting private static final Logger LOG = - LoggerFactory.getLogger(ReconContainerDBProvider.class); + LoggerFactory.getLogger(ReconDBProvider.class); private OzoneConfiguration configuration; private ReconUtils reconUtils; @Inject - public ReconContainerDBProvider(OzoneConfiguration configuration, - ReconUtils reconUtils) { + public ReconDBProvider(OzoneConfiguration configuration, + ReconUtils reconUtils) { this.configuration = configuration; this.reconUtils = reconUtils; } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java index ccdc2799ea0b..efe2169b0a5e 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java @@ -42,7 +42,7 @@ import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerDBProvider; +import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; import org.junit.Assert; import org.junit.rules.TemporaryFolder; @@ -174,7 +174,7 @@ protected void configure() { if (withContainerDB) { bind(ContainerDBServiceProvider.class) .to(ContainerDBServiceProviderImpl.class).in(Singleton.class); - bind(DBStore.class).toProvider(ReconContainerDBProvider.class). + bind(DBStore.class).toProvider(ReconDBProvider.class). in(Singleton.class); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 03aa184b5f74..7179865897f8 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -458,7 +458,6 @@ public void testGetDatanodes() throws Exception { .findFirst().orElse(null); return (datanodeMetadata1 != null && datanodeMetadata1.getContainers() == 1 && - datanodeMetadata1.getOpenContainers() == 1 && reconScm.getPipelineManager() .getContainersInPipeline(pipeline.getId()).size() == 1); }); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerDBProvider.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java similarity index 88% rename from hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerDBProvider.java rename to hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java index ad1feeb68a32..5cdc66508a3b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerDBProvider.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java @@ -40,7 +40,7 @@ * Tests the class that provides the instance of the DB Store used by Recon to * store its container - key data. */ -public class TestReconContainerDBProvider { +public class TestReconDBProvider { @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); @@ -57,7 +57,7 @@ protected void configure() { OzoneConfiguration configuration = new OzoneConfiguration(); configuration.set(OZONE_RECON_DB_DIR, dbDir.getAbsolutePath()); bind(OzoneConfiguration.class).toInstance(configuration); - bind(DBStore.class).toProvider(ReconContainerDBProvider.class).in( + bind(DBStore.class).toProvider(ReconDBProvider.class).in( Singleton.class); } }); @@ -65,9 +65,9 @@ protected void configure() { @Test public void testGet() throws Exception { - ReconContainerDBProvider reconContainerDBProvider = injector.getInstance( - ReconContainerDBProvider.class); - DBStore dbStore = reconContainerDBProvider.get(); + ReconDBProvider reconDBProvider = injector.getInstance( + ReconDBProvider.class); + DBStore dbStore = reconDBProvider.get(); assertNotNull(dbStore); } From 77ad2a5908d3604373ad7f9f00a8648cb231926c Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Tue, 22 Jun 2021 17:08:05 -0400 Subject: [PATCH 02/15] rename container/NSSummary service providers, added a ReconRocksDB class to manage DB-level operations (cherry picked from commit 5f35ce66cb9b7da33d3ca065a96d90371197ecb7) --- .../recon/TestReconWithOzoneManagerHA.java | 6 +- .../hadoop/ozone/recon/ReconConstants.java | 1 + .../ozone/recon/ReconControllerModule.java | 8 +- .../hadoop/ozone/recon/ReconServer.java | 16 +- .../ozone/recon/api/ContainerEndpoint.java | 12 +- .../ozone/recon/api/types/NSSummary.java | 13 +- .../ozone/recon/codec/NSSummaryCodec.java | 37 ++++- .../recon/scm/ReconContainerManager.java | 8 +- .../ReconStorageContainerManagerFacade.java | 6 +- .../recon/spi/NSSummaryServiceProvider.java | 5 - ...ava => ReconContainerMetadataManager.java} | 2 +- .../spi/ReconNSSummaryMetadataManager.java | 5 + .../impl/NSSummaryServiceProviderImpl.java | 9 -- ...=> ReconContainerMetadataManagerImpl.java} | 12 +- .../recon/spi/impl/ReconDBDefinition.java | 8 +- .../ReconNSSummaryMetadataManagerImpl.java | 8 + .../ozone/recon/spi/impl/ReconRocksDB.java | 35 +++++ .../recon/tasks/ContainerKeyMapperTask.java | 32 ++-- .../hadoop/ozone/recon/ReconTestInjector.java | 8 +- .../recon/api/TestContainerEndpoint.java | 10 +- .../AbstractReconContainerManagerTest.java | 4 +- ...estReconContainerMetadataManagerImpl.java} | 140 +++++++++--------- .../tasks/TestContainerKeyMapperTask.java | 58 ++++---- 23 files changed, 254 insertions(+), 189 deletions(-) delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/NSSummaryServiceProvider.java rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/{ContainerDBServiceProvider.java => ReconContainerMetadataManager.java} (99%) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNSSummaryMetadataManager.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/NSSummaryServiceProviderImpl.java rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/{ContainerDBServiceProviderImpl.java => ReconContainerMetadataManagerImpl.java} (97%) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNSSummaryMetadataManagerImpl.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java rename hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/{TestContainerDBServiceProviderImpl.java => TestReconContainerMetadataManagerImpl.java} (65%) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java index 8123410344e6..fc89af3bfe0e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java @@ -38,7 +38,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.ozone.test.GenericTestUtils; import org.junit.After; @@ -130,10 +130,10 @@ public void testReconGetsSnapshotFromLeader() throws Exception { // Sync data to Recon impl.syncDataFromOM(); - ContainerDBServiceProvider containerDBServiceProvider = + ReconContainerMetadataManager reconContainerMetadataManager = cluster.getReconServer().getContainerDBServiceProvider(); TableIterator iterator = - containerDBServiceProvider.getContainerTableIterator(); + reconContainerMetadataManager.getContainerTableIterator(); String reconKeyPrefix = null; while (iterator.hasNext()) { Table.KeyValue keyValue = diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index 0cbc61fa2459..b4cfcf291e76 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -58,4 +58,5 @@ private ReconConstants() { + PIPELINE_DB_SUFFIX; public static final String RECON_SCM_NODE_DB = "recon-node.db"; + public static final int NUM_OF_BINS = 40; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index 81bbe4391b49..c8435d330d11 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -38,10 +38,10 @@ import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.recovery.ReconOmMetadataManagerImpl; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerMetadataManagerImpl; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; @@ -92,8 +92,8 @@ protected void configure() { bind(OMMetadataManager.class).to(ReconOmMetadataManagerImpl.class); bind(ContainerHealthSchemaManager.class).in(Singleton.class); - bind(ContainerDBServiceProvider.class) - .to(ContainerDBServiceProviderImpl.class).in(Singleton.class); + bind(ReconContainerMetadataManager.class) + .to(ReconContainerMetadataManagerImpl.class).in(Singleton.class); bind(OzoneManagerServiceProvider.class) .to(OzoneManagerServiceProviderImpl.class).in(Singleton.class); bind(ReconUtils.class).in(Singleton.class); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index c2b1c5fdd1bd..af90095e6a59 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.ozone.OzoneSecurityUtil; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.util.OzoneVersionInfo; @@ -56,7 +56,7 @@ public class ReconServer extends GenericCli { private Injector injector; private ReconHttpServer httpServer; - private ContainerDBServiceProvider containerDBServiceProvider; + private ReconContainerMetadataManager reconContainerMetadataManager; private OzoneManagerServiceProvider ozoneManagerServiceProvider; private OzoneStorageContainerManager reconStorageContainerManager; private OzoneConfiguration configuration; @@ -93,8 +93,8 @@ protected void configureServlets() { LOG.info("Initializing Recon server..."); try { loginReconUserIfSecurityEnabled(configuration); - this.containerDBServiceProvider = - injector.getInstance(ContainerDBServiceProvider.class); + this.reconContainerMetadataManager = + injector.getInstance(ReconContainerMetadataManager.class); ReconSchemaManager reconSchemaManager = injector.getInstance(ReconSchemaManager.class); @@ -159,8 +159,8 @@ public void stop() throws Exception { if (ozoneManagerServiceProvider != null) { ozoneManagerServiceProvider.stop(); } - if (containerDBServiceProvider != null) { - containerDBServiceProvider.close(); + if (reconContainerMetadataManager != null) { + reconContainerMetadataManager.close(); } isStarted = false; } @@ -234,8 +234,8 @@ public StorageContainerServiceProvider getStorageContainerServiceProvider() { } @VisibleForTesting - public ContainerDBServiceProvider getContainerDBServiceProvider() { - return containerDBServiceProvider; + public ReconContainerMetadataManager getContainerDBServiceProvider() { + return reconContainerMetadataManager; } @VisibleForTesting diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java index 93f41fc7f5f9..8eb5ea807fac 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java @@ -59,7 +59,7 @@ import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.scm.ReconContainerManager; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition.UnHealthyContainerStates; import org.hadoop.ozone.recon.schema.tables.pojos.UnhealthyContainers; @@ -79,7 +79,7 @@ public class ContainerEndpoint { @Inject - private ContainerDBServiceProvider containerDBServiceProvider; + private ReconContainerMetadataManager reconContainerMetadataManager; @Inject private ReconOMMetadataManager omMetadataManager; @@ -113,8 +113,8 @@ public Response getContainers( Map containersMap; long containersCount; try { - containersMap = containerDBServiceProvider.getContainers(limit, prevKey); - containersCount = containerDBServiceProvider.getCountForContainers(); + containersMap = reconContainerMetadataManager.getContainers(limit, prevKey); + containersCount = reconContainerMetadataManager.getCountForContainers(); } catch (IOException ioEx) { throw new WebApplicationException(ioEx, Response.Status.INTERNAL_SERVER_ERROR); @@ -147,7 +147,7 @@ public Response getKeysForContainer( long totalCount; try { Map containerKeyPrefixMap = - containerDBServiceProvider.getKeyPrefixesForContainer(containerID, + reconContainerMetadataManager.getKeyPrefixesForContainer(containerID, prevKeyPrefix); // Get set of Container-Key mappings for given containerId. @@ -204,7 +204,7 @@ public Response getKeysForContainer( } totalCount = - containerDBServiceProvider.getKeyCountForContainer(containerID); + reconContainerMetadataManager.getKeyCountForContainer(containerID); } catch (IOException ioEx) { throw new WebApplicationException(ioEx, Response.Status.INTERNAL_SERVER_ERROR); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java index aec86e27675e..4118063d7d5a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java @@ -1,5 +1,7 @@ package org.apache.hadoop.ozone.recon.api.types; +import org.apache.hadoop.ozone.recon.ReconConstants; + import java.util.Arrays; /** @@ -14,12 +16,8 @@ public class NSSummary { public NSSummary() { this.numOfFiles = 0; this.sizeOfFiles = 0; - // < 1KB, 1KB - 512KB, 513KB - 1024KB, 1MB - 10MB, 10MB - 100MB, - // 100MB - 1000MB, 1GB - 10GB, 10GB - 100GB, 100GB - 1000GB, > 1TB - // TODO: can we generate a more reasonable bucket - // Should this file size distribution be fixed or dynamic based on - // the actual situation? - this.fileSizeBucket = new int[10]; + // TODO: I read the min is 1024(2^10), max is 1PB(2^50), so the number of buckets should be 40? + this.fileSizeBucket = new int[ReconConstants.NUM_OF_BINS]; } public int getNumOfFiles() { @@ -43,6 +41,7 @@ public void setSizeOfFiles(int sizeOfFiles) { } public void setFileSizeBucket(int[] fileSizeBucket) { - this.fileSizeBucket = Arrays.copyOf(fileSizeBucket, 10); + this.fileSizeBucket = Arrays.copyOf( + fileSizeBucket, ReconConstants.NUM_OF_BINS); } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index 8e0c6aba81a5..6a42eb882137 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -1,24 +1,55 @@ package org.apache.hadoop.ozone.recon.codec; +import org.apache.hadoop.hdds.utils.db.IntegerCodec; +import org.apache.hadoop.ozone.recon.ReconConstants; import org.apache.hadoop.ozone.recon.api.types.NSSummary; import org.apache.hadoop.hdds.utils.db.Codec; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; import java.io.IOException; public class NSSummaryCodec implements Codec{ + private final Codec integerCodec = new IntegerCodec(); + // 2 int fields + 40-length int array + private static final int NUM_OF_INTS = 2 + ReconConstants.NUM_OF_BINS; + @Override public byte[] toPersistedFormat(NSSummary object) throws IOException { - return new byte[0]; + final int sizeOfRes = NUM_OF_INTS * Integer.BYTES; + ByteArrayOutputStream out = new ByteArrayOutputStream(sizeOfRes); + out.write(integerCodec.toPersistedFormat(object.getNumOfFiles())); + out.write(integerCodec.toPersistedFormat(object.getSizeOfFiles())); + int[] fileSizeBucket = object.getFileSizeBucket(); + for (int i = 0; i < ReconConstants.NUM_OF_BINS; ++i) { + out.write(integerCodec.toPersistedFormat(fileSizeBucket[i])); + } + return out.toByteArray(); } @Override public NSSummary fromPersistedFormat(byte[] rawData) throws IOException { - return null; + assert(rawData.length == NUM_OF_INTS); + DataInputStream in = new DataInputStream(new ByteArrayInputStream(rawData)); + NSSummary res = new NSSummary(); + res.setNumOfFiles(in.readInt()); + res.setNumOfFiles(in.readInt()); + int[] fileSizeBucket = new int[ReconConstants.NUM_OF_BINS]; + for (int i = 0; i < ReconConstants.NUM_OF_BINS && in.available() > 0; ++i) { + fileSizeBucket[i] = in.readInt(); + } + res.setFileSizeBucket(fileSizeBucket); + return res; } @Override public NSSummary copyObject(NSSummary object) { - return null; + NSSummary copy = new NSSummary(); + copy.setNumOfFiles(object.getNumOfFiles()); + copy.setSizeOfFiles(object.getSizeOfFiles()); + copy.setFileSizeBucket(object.getFileSizeBucket()); + return copy; } } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java index e2958938623d..215ceb1d5f71 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java @@ -50,7 +50,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.recon.persistence.ContainerHistory; import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.slf4j.Logger; @@ -66,7 +66,7 @@ public class ReconContainerManager extends ContainerManagerImpl { private StorageContainerServiceProvider scmClient; private PipelineManager pipelineManager; private final ContainerHealthSchemaManager containerHealthSchemaManager; - private final ContainerDBServiceProvider cdbServiceProvider; + private final ReconContainerMetadataManager cdbServiceProvider; private final Table nodeDB; // Container ID -> Datanode UUID -> Timestamp private final Map> replicaHistoryMap; @@ -91,7 +91,7 @@ public ReconContainerManager( PipelineManager pipelineManager, StorageContainerServiceProvider scm, ContainerHealthSchemaManager containerHealthSchemaManager, - ContainerDBServiceProvider containerDBServiceProvider, + ReconContainerMetadataManager reconContainerMetadataManager, SCMHAManager scmhaManager, SequenceIdGenerator sequenceIdGen) throws IOException { @@ -99,7 +99,7 @@ public ReconContainerManager( this.scmClient = scm; this.pipelineManager = pipelineManager; this.containerHealthSchemaManager = containerHealthSchemaManager; - this.cdbServiceProvider = containerDBServiceProvider; + this.cdbServiceProvider = reconContainerMetadataManager; // batchHandler = scmDBStore this.nodeDB = ReconSCMDBDefinition.NODES.getTable(store); this.replicaHistoryMap = new ConcurrentHashMap<>(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java index 652ab6e5e1a6..87c4eef121ff 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java @@ -63,7 +63,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.recon.fsck.ContainerHealthTask; import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.tasks.ReconTaskConfig; import com.google.inject.Inject; @@ -108,7 +108,7 @@ public ReconStorageContainerManagerFacade(OzoneConfiguration conf, StorageContainerServiceProvider scmServiceProvider, ReconTaskStatusDao reconTaskStatusDao, ContainerHealthSchemaManager containerHealthSchemaManager, - ContainerDBServiceProvider containerDBServiceProvider) + ReconContainerMetadataManager reconContainerMetadataManager) throws IOException { reconNodeDetails = getReconNodeDetails(conf); this.eventQueue = new EventQueue(); @@ -143,7 +143,7 @@ public ReconStorageContainerManagerFacade(OzoneConfiguration conf, dbStore, ReconSCMDBDefinition.CONTAINERS.getTable(dbStore), pipelineManager, scmServiceProvider, - containerHealthSchemaManager, containerDBServiceProvider, + containerHealthSchemaManager, reconContainerMetadataManager, scmhaManager, sequenceIdGen); this.scmServiceProvider = scmServiceProvider; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/NSSummaryServiceProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/NSSummaryServiceProvider.java deleted file mode 100644 index 78b7c68b699f..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/NSSummaryServiceProvider.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.apache.hadoop.ozone.recon.spi; - -public interface NSSummaryServiceProvider { - -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java similarity index 99% rename from hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java rename to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java index df771a625e2b..4f122b1041a5 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java @@ -32,7 +32,7 @@ * The Recon Container DB Service interface. */ @InterfaceStability.Unstable -public interface ContainerDBServiceProvider { +public interface ReconContainerMetadataManager { /* * Close the container DB diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNSSummaryMetadataManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNSSummaryMetadataManager.java new file mode 100644 index 000000000000..184d78de4e40 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNSSummaryMetadataManager.java @@ -0,0 +1,5 @@ +package org.apache.hadoop.ozone.recon.spi; + +public interface ReconNSSummaryMetadataManager { + +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/NSSummaryServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/NSSummaryServiceProviderImpl.java deleted file mode 100644 index 6b259429ffb2..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/NSSummaryServiceProviderImpl.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.apache.hadoop.ozone.recon.spi.impl; - -import com.google.inject.Provider; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.ozone.recon.spi.NSSummaryServiceProvider; - -public class NSSummaryServiceProviderImpl implements NSSummaryServiceProvider { - -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java similarity index 97% rename from hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java rename to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java index c3e4e50f38ba..c273c00ce94f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java @@ -45,7 +45,7 @@ import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata; import org.apache.hadoop.ozone.recon.scm.ContainerReplicaHistory; import org.apache.hadoop.ozone.recon.scm.ContainerReplicaHistoryList; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; @@ -60,11 +60,11 @@ * Implementation of the Recon Container DB Service. */ @Singleton -public class ContainerDBServiceProviderImpl - implements ContainerDBServiceProvider { +public class ReconContainerMetadataManagerImpl + implements ReconContainerMetadataManager { private static final Logger LOG = - LoggerFactory.getLogger(ContainerDBServiceProviderImpl.class); + LoggerFactory.getLogger(ReconContainerMetadataManagerImpl.class); private Table containerKeyTable; private Table containerKeyCountTable; @@ -82,8 +82,8 @@ public class ContainerDBServiceProviderImpl private Configuration sqlConfiguration; @Inject - public ContainerDBServiceProviderImpl(DBStore dbStore, - Configuration sqlConfiguration) { + public ReconContainerMetadataManagerImpl(DBStore dbStore, + Configuration sqlConfiguration) { containerDbStore = dbStore; globalStatsDao = new GlobalStatsDao(sqlConfiguration); initializeTables(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java index a3ce33214a5f..ed1ed5fd7d58 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java @@ -64,11 +64,11 @@ public ReconDBDefinition(String dbName) { ContainerReplicaHistoryList.class, new ContainerReplicaHistoryListCodec()); - public static final DBColumnFamilyDefinition - NAMESPACE_SUMMARY = new DBColumnFamilyDefinition( + public static final DBColumnFamilyDefinition + NAMESPACE_SUMMARY = new DBColumnFamilyDefinition( "namespaceSummaryTable", - String.class, - new StringCodec(), + Long.class, + new LongCodec(), NSSummary.class, new NSSummaryCodec()); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNSSummaryMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNSSummaryMetadataManagerImpl.java new file mode 100644 index 000000000000..cab88ef746ea --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNSSummaryMetadataManagerImpl.java @@ -0,0 +1,8 @@ +package org.apache.hadoop.ozone.recon.spi.impl; + +import org.apache.hadoop.ozone.recon.spi.ReconNSSummaryMetadataManager; + +public class ReconNSSummaryMetadataManagerImpl + implements ReconNSSummaryMetadataManager { + +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java new file mode 100644 index 000000000000..a6e30db5e014 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java @@ -0,0 +1,35 @@ +package org.apache.hadoop.ozone.recon.spi.impl; +import javax.inject.Inject; +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.scm.ContainerReplicaHistoryList; + + +public class ReconRocksDB { + @Inject + ReconRocksDB(DBStore dbStore) { + + } + + public Table getContainerKeyTable() { + return null; + } + + public Table getContainerKeyCountTable() { + return null; + } + + public Table getContainerReplicaTable() { + return null; + } + + public Table getNSSummaryTable() { + return null; + } + + public void close() { + + } +} \ No newline at end of file diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java index af38d5298a98..85e29cd1d74b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java @@ -37,7 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.slf4j.Logger; @@ -54,12 +54,12 @@ public class ContainerKeyMapperTask implements ReconOmTask { private static final Logger LOG = LoggerFactory.getLogger(ContainerKeyMapperTask.class); - private ContainerDBServiceProvider containerDBServiceProvider; + private ReconContainerMetadataManager reconContainerMetadataManager; @Inject - public ContainerKeyMapperTask(ContainerDBServiceProvider - containerDBServiceProvider) { - this.containerDBServiceProvider = containerDBServiceProvider; + public ContainerKeyMapperTask(ReconContainerMetadataManager + reconContainerMetadataManager) { + this.reconContainerMetadataManager = reconContainerMetadataManager; } /** @@ -74,7 +74,7 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { Instant start = Instant.now(); // initialize new container DB - containerDBServiceProvider.initNewContainerDB(new HashMap<>()); + reconContainerMetadataManager.initNewContainerDB(new HashMap<>()); Table omKeyInfoTable = omMetadataManager.getKeyTable(); try (TableIterator> @@ -170,7 +170,7 @@ private void deleteOMKeyFromContainerDB(String key) TableIterator> containerIterator = - containerDBServiceProvider.getContainerTableIterator(); + reconContainerMetadataManager.getContainerTableIterator(); Set keysToBeDeleted = new HashSet<>(); @@ -184,14 +184,14 @@ private void deleteOMKeyFromContainerDB(String key) } for (ContainerKeyPrefix containerKeyPrefix : keysToBeDeleted) { - containerDBServiceProvider.deleteContainerMapping(containerKeyPrefix); + reconContainerMetadataManager.deleteContainerMapping(containerKeyPrefix); // decrement count and update containerKeyCount. Long containerID = containerKeyPrefix.getContainerId(); long keyCount = - containerDBServiceProvider.getKeyCountForContainer(containerID); + reconContainerMetadataManager.getKeyCountForContainer(containerID); if (keyCount > 0) { - containerDBServiceProvider.storeContainerKeyCount(containerID, + reconContainerMetadataManager.storeContainerKeyCount(containerID, --keyCount); } } @@ -216,34 +216,34 @@ private void writeOMKeyToContainerDB(String key, OmKeyInfo omKeyInfo) long containerId = omKeyLocationInfo.getContainerID(); ContainerKeyPrefix containerKeyPrefix = new ContainerKeyPrefix( containerId, key, keyVersion); - if (containerDBServiceProvider.getCountForContainerKeyPrefix( + if (reconContainerMetadataManager.getCountForContainerKeyPrefix( containerKeyPrefix) == 0) { // Save on writes. No need to save same container-key prefix // mapping again. - containerDBServiceProvider.storeContainerKeyMapping( + reconContainerMetadataManager.storeContainerKeyMapping( containerKeyPrefix, 1); // check if container already exists and // increment the count of containers if it does not exist - if (!containerDBServiceProvider.doesContainerExists(containerId)) { + if (!reconContainerMetadataManager.doesContainerExists(containerId)) { containerCountToIncrement++; } // update the count of keys for the given containerID long keyCount = - containerDBServiceProvider.getKeyCountForContainer(containerId); + reconContainerMetadataManager.getKeyCountForContainer(containerId); // increment the count and update containerKeyCount. // keyCount will be 0 if containerID is not found. So, there is no // need to initialize keyCount for the first time. - containerDBServiceProvider.storeContainerKeyCount(containerId, + reconContainerMetadataManager.storeContainerKeyCount(containerId, ++keyCount); } } } if (containerCountToIncrement > 0) { - containerDBServiceProvider + reconContainerMetadataManager .incrementContainerCountBy(containerCountToIncrement); } } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java index efe2169b0a5e..ddbb17bcaced 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java @@ -39,9 +39,9 @@ import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.ContainerDBServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerMetadataManagerImpl; import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; import org.junit.Assert; import org.junit.rules.TemporaryFolder; @@ -172,8 +172,8 @@ protected void configure() { } if (withContainerDB) { - bind(ContainerDBServiceProvider.class) - .to(ContainerDBServiceProviderImpl.class).in(Singleton.class); + bind(ReconContainerMetadataManager.class) + .to(ReconContainerMetadataManagerImpl.class).in(Singleton.class); bind(DBStore.class).toProvider(ReconDBProvider.class). in(Singleton.class); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java index 9e4afd2ca3fd..bbf5daa5451c 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java @@ -75,7 +75,7 @@ import org.apache.hadoop.ozone.recon.scm.ReconContainerManager; import org.apache.hadoop.ozone.recon.scm.ReconPipelineManager; import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; @@ -101,7 +101,7 @@ public class TestContainerEndpoint { private OzoneStorageContainerManager ozoneStorageContainerManager; private ReconContainerManager reconContainerManager; private ReconPipelineManager reconPipelineManager; - private ContainerDBServiceProvider containerDbServiceProvider; + private ReconContainerMetadataManager reconContainerMetadataManager; private ContainerEndpoint containerEndpoint; private boolean isSetupDone = false; private ContainerHealthSchemaManager containerHealthSchemaManager; @@ -143,8 +143,8 @@ private void initializeInjector() throws Exception { ozoneStorageContainerManager.getContainerManager(); reconPipelineManager = (ReconPipelineManager) ozoneStorageContainerManager.getPipelineManager(); - containerDbServiceProvider = - reconTestInjector.getInstance(ContainerDBServiceProvider.class); + reconContainerMetadataManager = + reconTestInjector.getInstance(ReconContainerMetadataManager.class); containerEndpoint = reconTestInjector.getInstance(ContainerEndpoint.class); containerHealthSchemaManager = reconTestInjector.getInstance(ContainerHealthSchemaManager.class); @@ -229,7 +229,7 @@ public void setUp() throws Exception { when(tableMock.getName()).thenReturn("KeyTable"); when(omMetadataManagerMock.getKeyTable()).thenReturn(tableMock); ContainerKeyMapperTask containerKeyMapperTask = - new ContainerKeyMapperTask(containerDbServiceProvider); + new ContainerKeyMapperTask(reconContainerMetadataManager); containerKeyMapperTask.reprocess(reconOMMetadataManager); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java index 1276b770178e..c1218dc80089 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/AbstractReconContainerManagerTest.java @@ -44,7 +44,7 @@ import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.OPEN; @@ -109,7 +109,7 @@ public void setUp() throws Exception { pipelineManager, getScmServiceProvider(), mock(ContainerHealthSchemaManager.class), - mock(ContainerDBServiceProvider.class), + mock(ReconContainerMetadataManager.class), scmhaManager, sequenceIdGen); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java similarity index 65% rename from hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java rename to hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java index a4b0b54da8ed..6accb23154f0 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestContainerDBServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java @@ -29,7 +29,7 @@ import org.apache.hadoop.ozone.recon.ReconTestInjector; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -40,11 +40,11 @@ /** * Unit Tests for ContainerDBServiceProviderImpl. */ -public class TestContainerDBServiceProviderImpl { +public class TestReconContainerMetadataManagerImpl { @ClassRule public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); - private static ContainerDBServiceProvider containerDbServiceProvider; + private static ReconContainerMetadataManager reconContainerMetadataManager; private String keyPrefix1 = "V3/B1/K1"; private String keyPrefix2 = "V3/B1/K2"; @@ -57,14 +57,14 @@ public static void setupOnce() throws Exception { .withReconSqlDb() .withContainerDB() .build(); - containerDbServiceProvider = - reconTestInjector.getInstance(ContainerDBServiceProvider.class); + reconContainerMetadataManager = + reconTestInjector.getInstance(ReconContainerMetadataManager.class); } @Before public void setUp() throws Exception { // Reset containerDB before running each test - containerDbServiceProvider.initNewContainerDB(null); + reconContainerMetadataManager.initNewContainerDB(null); } private void populateKeysInContainers(long containerId1, long containerId2) @@ -72,18 +72,18 @@ private void populateKeysInContainers(long containerId1, long containerId2) ContainerKeyPrefix containerKeyPrefix1 = new ContainerKeyPrefix(containerId1, keyPrefix1, 0); - containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix1, + reconContainerMetadataManager.storeContainerKeyMapping(containerKeyPrefix1, 1); ContainerKeyPrefix containerKeyPrefix2 = new ContainerKeyPrefix( containerId1, keyPrefix2, 0); - containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix2, + reconContainerMetadataManager.storeContainerKeyMapping(containerKeyPrefix2, 2); ContainerKeyPrefix containerKeyPrefix3 = new ContainerKeyPrefix( containerId2, keyPrefix3, 0); - containerDbServiceProvider.storeContainerKeyMapping(containerKeyPrefix3, + reconContainerMetadataManager.storeContainerKeyMapping(containerKeyPrefix3, 3); } @@ -106,11 +106,11 @@ public void testInitNewContainerDB() throws Exception { for (Map.Entry entry : prefixCounts.entrySet()) { - containerDbServiceProvider.storeContainerKeyMapping( + reconContainerMetadataManager.storeContainerKeyMapping( entry.getKey(), prefixCounts.get(entry.getKey())); } - assertEquals(1, containerDbServiceProvider + assertEquals(1, reconContainerMetadataManager .getCountForContainerKeyPrefix(ckp1).intValue()); prefixCounts.clear(); @@ -123,9 +123,9 @@ public void testInitNewContainerDB() throws Exception { "V1/B3/K2", 0); prefixCounts.put(ckp5, 15); - containerDbServiceProvider.initNewContainerDB(prefixCounts); + reconContainerMetadataManager.initNewContainerDB(prefixCounts); Map keyPrefixesForContainer = - containerDbServiceProvider.getKeyPrefixesForContainer(containerId); + reconContainerMetadataManager.getKeyPrefixesForContainer(containerId); assertEquals(4, keyPrefixesForContainer.size()); assertEquals(12, keyPrefixesForContainer.get(ckp2).intValue()); @@ -133,7 +133,7 @@ public void testInitNewContainerDB() throws Exception { assertEquals(14, keyPrefixesForContainer.get(ckp4).intValue()); assertEquals(15, keyPrefixesForContainer.get(ckp5).intValue()); - assertEquals(0, containerDbServiceProvider + assertEquals(0, reconContainerMetadataManager .getCountForContainerKeyPrefix(ckp1).intValue()); } @@ -149,20 +149,20 @@ public void testStoreContainerKeyMapping() throws Exception { for (Map.Entry entry : prefixCounts.entrySet()) { ContainerKeyPrefix containerKeyPrefix = new ContainerKeyPrefix( containerId, entry.getKey(), 0); - containerDbServiceProvider.storeContainerKeyMapping( + reconContainerMetadataManager.storeContainerKeyMapping( containerKeyPrefix, prefixCounts.get(entry.getKey())); } Assert.assertEquals(1, - containerDbServiceProvider.getCountForContainerKeyPrefix( + reconContainerMetadataManager.getCountForContainerKeyPrefix( new ContainerKeyPrefix(containerId, keyPrefix1, 0)).longValue()); Assert.assertEquals(2, - containerDbServiceProvider.getCountForContainerKeyPrefix( + reconContainerMetadataManager.getCountForContainerKeyPrefix( new ContainerKeyPrefix(containerId, keyPrefix2, 0)).longValue()); Assert.assertEquals(3, - containerDbServiceProvider.getCountForContainerKeyPrefix( + reconContainerMetadataManager.getCountForContainerKeyPrefix( new ContainerKeyPrefix(containerId, keyPrefix3, 0)).longValue()); } @@ -171,61 +171,61 @@ public void testStoreContainerKeyMapping() throws Exception { public void testStoreContainerKeyCount() throws Exception { long containerId = 1L; long nextContainerId = 2L; - containerDbServiceProvider.storeContainerKeyCount(containerId, 2L); - containerDbServiceProvider.storeContainerKeyCount(nextContainerId, 3L); + reconContainerMetadataManager.storeContainerKeyCount(containerId, 2L); + reconContainerMetadataManager.storeContainerKeyCount(nextContainerId, 3L); assertEquals(2, - containerDbServiceProvider.getKeyCountForContainer(containerId)); + reconContainerMetadataManager.getKeyCountForContainer(containerId)); assertEquals(3, - containerDbServiceProvider.getKeyCountForContainer(nextContainerId)); + reconContainerMetadataManager.getKeyCountForContainer(nextContainerId)); - containerDbServiceProvider.storeContainerKeyCount(containerId, 20L); + reconContainerMetadataManager.storeContainerKeyCount(containerId, 20L); assertEquals(20, - containerDbServiceProvider.getKeyCountForContainer(containerId)); + reconContainerMetadataManager.getKeyCountForContainer(containerId)); } @Test public void testGetKeyCountForContainer() throws Exception { long containerId = 1L; long nextContainerId = 2L; - containerDbServiceProvider.storeContainerKeyCount(containerId, 2L); - containerDbServiceProvider.storeContainerKeyCount(nextContainerId, 3L); + reconContainerMetadataManager.storeContainerKeyCount(containerId, 2L); + reconContainerMetadataManager.storeContainerKeyCount(nextContainerId, 3L); assertEquals(2, - containerDbServiceProvider.getKeyCountForContainer(containerId)); + reconContainerMetadataManager.getKeyCountForContainer(containerId)); assertEquals(3, - containerDbServiceProvider.getKeyCountForContainer(nextContainerId)); + reconContainerMetadataManager.getKeyCountForContainer(nextContainerId)); assertEquals(0, - containerDbServiceProvider.getKeyCountForContainer(5L)); + reconContainerMetadataManager.getKeyCountForContainer(5L)); } @Test public void testDoesContainerExists() throws Exception { long containerId = 1L; long nextContainerId = 2L; - containerDbServiceProvider.storeContainerKeyCount(containerId, 2L); - containerDbServiceProvider.storeContainerKeyCount(nextContainerId, 3L); + reconContainerMetadataManager.storeContainerKeyCount(containerId, 2L); + reconContainerMetadataManager.storeContainerKeyCount(nextContainerId, 3L); - assertTrue(containerDbServiceProvider.doesContainerExists(containerId)); - assertTrue(containerDbServiceProvider.doesContainerExists(nextContainerId)); - assertFalse(containerDbServiceProvider.doesContainerExists(0L)); - assertFalse(containerDbServiceProvider.doesContainerExists(3L)); + assertTrue(reconContainerMetadataManager.doesContainerExists(containerId)); + assertTrue(reconContainerMetadataManager.doesContainerExists(nextContainerId)); + assertFalse(reconContainerMetadataManager.doesContainerExists(0L)); + assertFalse(reconContainerMetadataManager.doesContainerExists(3L)); } @Test public void testGetCountForContainerKeyPrefix() throws Exception { long containerId = System.currentTimeMillis(); - containerDbServiceProvider.storeContainerKeyMapping(new + reconContainerMetadataManager.storeContainerKeyMapping(new ContainerKeyPrefix(containerId, keyPrefix1), 2); - Integer count = containerDbServiceProvider. + Integer count = reconContainerMetadataManager. getCountForContainerKeyPrefix(new ContainerKeyPrefix(containerId, keyPrefix1)); assertEquals(2L, count.longValue()); - count = containerDbServiceProvider. + count = reconContainerMetadataManager. getCountForContainerKeyPrefix(new ContainerKeyPrefix(containerId, "invalid")); assertEquals(0L, count.longValue()); @@ -246,13 +246,13 @@ public void testGetKeyPrefixesForContainer() throws Exception { Map keyPrefixMap = - containerDbServiceProvider.getKeyPrefixesForContainer(containerId); + reconContainerMetadataManager.getKeyPrefixesForContainer(containerId); assertEquals(2, keyPrefixMap.size()); assertEquals(1, keyPrefixMap.get(containerKeyPrefix1).longValue()); assertEquals(2, keyPrefixMap.get(containerKeyPrefix2).longValue()); - keyPrefixMap = containerDbServiceProvider.getKeyPrefixesForContainer( + keyPrefixMap = reconContainerMetadataManager.getKeyPrefixesForContainer( nextContainerId); assertEquals(1, keyPrefixMap.size()); assertEquals(3, keyPrefixMap.get(containerKeyPrefix3).longValue()); @@ -268,25 +268,25 @@ public void testGetKeyPrefixesForContainerWithKeyPrefix() throws Exception { containerId, keyPrefix2, 0); Map keyPrefixMap = - containerDbServiceProvider.getKeyPrefixesForContainer(containerId, + reconContainerMetadataManager.getKeyPrefixesForContainer(containerId, keyPrefix1); assertEquals(1, keyPrefixMap.size()); assertEquals(2, keyPrefixMap.get(containerKeyPrefix2).longValue()); - keyPrefixMap = containerDbServiceProvider.getKeyPrefixesForContainer( + keyPrefixMap = reconContainerMetadataManager.getKeyPrefixesForContainer( nextContainerId, keyPrefix3); assertEquals(0, keyPrefixMap.size()); // test for negative cases - keyPrefixMap = containerDbServiceProvider.getKeyPrefixesForContainer( + keyPrefixMap = reconContainerMetadataManager.getKeyPrefixesForContainer( containerId, "V3/B1/invalid"); assertEquals(0, keyPrefixMap.size()); - keyPrefixMap = containerDbServiceProvider.getKeyPrefixesForContainer( + keyPrefixMap = reconContainerMetadataManager.getKeyPrefixesForContainer( containerId, keyPrefix3); assertEquals(0, keyPrefixMap.size()); - keyPrefixMap = containerDbServiceProvider.getKeyPrefixesForContainer( + keyPrefixMap = reconContainerMetadataManager.getKeyPrefixesForContainer( 10L, ""); assertEquals(0, keyPrefixMap.size()); } @@ -298,35 +298,35 @@ public void testGetContainersWithPrevContainer() throws Exception { populateKeysInContainers(containerId, nextContainerId); Map containerMap = - containerDbServiceProvider.getContainers(-1, 0L); + reconContainerMetadataManager.getContainers(-1, 0L); assertEquals(2, containerMap.size()); assertEquals(3, containerMap.get(containerId).getNumberOfKeys()); assertEquals(3, containerMap.get(nextContainerId).getNumberOfKeys()); // test if limit works - containerMap = containerDbServiceProvider.getContainers( + containerMap = reconContainerMetadataManager.getContainers( 1, 0L); assertEquals(1, containerMap.size()); assertNull(containerMap.get(nextContainerId)); // test for prev key - containerMap = containerDbServiceProvider.getContainers( + containerMap = reconContainerMetadataManager.getContainers( -1, containerId); assertEquals(1, containerMap.size()); // containerId must be skipped from containerMap result assertNull(containerMap.get(containerId)); - containerMap = containerDbServiceProvider.getContainers( + containerMap = reconContainerMetadataManager.getContainers( -1, nextContainerId); assertEquals(0, containerMap.size()); // test for negative cases - containerMap = containerDbServiceProvider.getContainers( + containerMap = reconContainerMetadataManager.getContainers( -1, 10L); assertEquals(0, containerMap.size()); - containerMap = containerDbServiceProvider.getContainers( + containerMap = reconContainerMetadataManager.getContainers( 0, containerId); assertEquals(0, containerMap.size()); } @@ -338,49 +338,49 @@ public void testDeleteContainerMapping() throws Exception { populateKeysInContainers(containerId, nextContainerId); Map keyPrefixMap = - containerDbServiceProvider.getKeyPrefixesForContainer(containerId); + reconContainerMetadataManager.getKeyPrefixesForContainer(containerId); assertEquals(2, keyPrefixMap.size()); - containerDbServiceProvider.deleteContainerMapping(new ContainerKeyPrefix( + reconContainerMetadataManager.deleteContainerMapping(new ContainerKeyPrefix( containerId, keyPrefix2, 0)); keyPrefixMap = - containerDbServiceProvider.getKeyPrefixesForContainer(containerId); + reconContainerMetadataManager.getKeyPrefixesForContainer(containerId); assertEquals(1, keyPrefixMap.size()); } @Test public void testGetCountForContainers() throws Exception { - assertEquals(0, containerDbServiceProvider.getCountForContainers()); + assertEquals(0, reconContainerMetadataManager.getCountForContainers()); - containerDbServiceProvider.storeContainerCount(5L); + reconContainerMetadataManager.storeContainerCount(5L); - assertEquals(5L, containerDbServiceProvider.getCountForContainers()); - containerDbServiceProvider.incrementContainerCountBy(1L); + assertEquals(5L, reconContainerMetadataManager.getCountForContainers()); + reconContainerMetadataManager.incrementContainerCountBy(1L); - assertEquals(6L, containerDbServiceProvider.getCountForContainers()); + assertEquals(6L, reconContainerMetadataManager.getCountForContainers()); - containerDbServiceProvider.storeContainerCount(10L); - assertEquals(10L, containerDbServiceProvider.getCountForContainers()); + reconContainerMetadataManager.storeContainerCount(10L); + assertEquals(10L, reconContainerMetadataManager.getCountForContainers()); } @Test public void testStoreContainerCount() throws Exception { - containerDbServiceProvider.storeContainerCount(3L); - assertEquals(3L, containerDbServiceProvider.getCountForContainers()); + reconContainerMetadataManager.storeContainerCount(3L); + assertEquals(3L, reconContainerMetadataManager.getCountForContainers()); - containerDbServiceProvider.storeContainerCount(5L); - assertEquals(5L, containerDbServiceProvider.getCountForContainers()); + reconContainerMetadataManager.storeContainerCount(5L); + assertEquals(5L, reconContainerMetadataManager.getCountForContainers()); } @Test public void testIncrementContainerCountBy() throws Exception { - assertEquals(0, containerDbServiceProvider.getCountForContainers()); + assertEquals(0, reconContainerMetadataManager.getCountForContainers()); - containerDbServiceProvider.incrementContainerCountBy(1L); - assertEquals(1L, containerDbServiceProvider.getCountForContainers()); + reconContainerMetadataManager.incrementContainerCountBy(1L); + assertEquals(1L, reconContainerMetadataManager.getCountForContainers()); - containerDbServiceProvider.incrementContainerCountBy(3L); - assertEquals(4L, containerDbServiceProvider.getCountForContainers()); + reconContainerMetadataManager.incrementContainerCountBy(3L); + assertEquals(4L, reconContainerMetadataManager.getCountForContainers()); } } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java index 0f87e29f7f7f..75584b24b345 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java @@ -45,7 +45,7 @@ import org.apache.hadoop.ozone.recon.ReconTestInjector; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; -import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; import org.apache.hadoop.hdds.utils.db.Table; import org.junit.Before; @@ -61,7 +61,7 @@ public class TestContainerKeyMapperTask { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - private ContainerDBServiceProvider containerDbServiceProvider; + private ReconContainerMetadataManager reconContainerMetadataManager; private OMMetadataManager omMetadataManager; private ReconOMMetadataManager reconOMMetadataManager; private OzoneManagerServiceProviderImpl ozoneManagerServiceProvider; @@ -81,18 +81,18 @@ public void setUp() throws Exception { .withOmServiceProvider(ozoneManagerServiceProvider) .withContainerDB() .build(); - containerDbServiceProvider = - reconTestInjector.getInstance(ContainerDBServiceProvider.class); + reconContainerMetadataManager = + reconTestInjector.getInstance(ReconContainerMetadataManager.class); } @Test public void testReprocessOMDB() throws Exception{ Map keyPrefixesForContainer = - containerDbServiceProvider.getKeyPrefixesForContainer(1); + reconContainerMetadataManager.getKeyPrefixesForContainer(1); assertTrue(keyPrefixesForContainer.isEmpty()); - keyPrefixesForContainer = containerDbServiceProvider + keyPrefixesForContainer = reconContainerMetadataManager .getKeyPrefixesForContainer(2); assertTrue(keyPrefixesForContainer.isEmpty()); @@ -120,11 +120,11 @@ public void testReprocessOMDB() throws Exception{ Collections.singletonList(omKeyLocationInfoGroup)); ContainerKeyMapperTask containerKeyMapperTask = - new ContainerKeyMapperTask(containerDbServiceProvider); + new ContainerKeyMapperTask(reconContainerMetadataManager); containerKeyMapperTask.reprocess(reconOMMetadataManager); keyPrefixesForContainer = - containerDbServiceProvider.getKeyPrefixesForContainer(1); + reconContainerMetadataManager.getKeyPrefixesForContainer(1); assertEquals(1, keyPrefixesForContainer.size()); String omKey = omMetadataManager.getOzoneKey("sampleVol", "bucketOne", "key_one"); @@ -134,7 +134,7 @@ public void testReprocessOMDB() throws Exception{ keyPrefixesForContainer.get(containerKeyPrefix).intValue()); keyPrefixesForContainer = - containerDbServiceProvider.getKeyPrefixesForContainer(2); + reconContainerMetadataManager.getKeyPrefixesForContainer(2); assertEquals(1, keyPrefixesForContainer.size()); containerKeyPrefix = new ContainerKeyPrefix(2, omKey, 0); @@ -142,21 +142,21 @@ public void testReprocessOMDB() throws Exception{ keyPrefixesForContainer.get(containerKeyPrefix).intValue()); // Test if container key counts are updated - assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(1L)); - assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(2L)); - assertEquals(0, containerDbServiceProvider.getKeyCountForContainer(3L)); + assertEquals(1, reconContainerMetadataManager.getKeyCountForContainer(1L)); + assertEquals(1, reconContainerMetadataManager.getKeyCountForContainer(2L)); + assertEquals(0, reconContainerMetadataManager.getKeyCountForContainer(3L)); // Test if container count is updated - assertEquals(2, containerDbServiceProvider.getCountForContainers()); + assertEquals(2, reconContainerMetadataManager.getCountForContainers()); } @Test public void testProcessOMEvents() throws IOException { Map keyPrefixesForContainer = - containerDbServiceProvider.getKeyPrefixesForContainer(1); + reconContainerMetadataManager.getKeyPrefixesForContainer(1); assertTrue(keyPrefixesForContainer.isEmpty()); - keyPrefixesForContainer = containerDbServiceProvider + keyPrefixesForContainer = reconContainerMetadataManager .getKeyPrefixesForContainer(2); assertTrue(keyPrefixesForContainer.isEmpty()); @@ -225,46 +225,46 @@ public void testProcessOMEvents() throws IOException { }}); ContainerKeyMapperTask containerKeyMapperTask = - new ContainerKeyMapperTask(containerDbServiceProvider); + new ContainerKeyMapperTask(reconContainerMetadataManager); containerKeyMapperTask.reprocess(reconOMMetadataManager); - keyPrefixesForContainer = containerDbServiceProvider + keyPrefixesForContainer = reconContainerMetadataManager .getKeyPrefixesForContainer(1); assertEquals(1, keyPrefixesForContainer.size()); - keyPrefixesForContainer = containerDbServiceProvider + keyPrefixesForContainer = reconContainerMetadataManager .getKeyPrefixesForContainer(2); assertTrue(keyPrefixesForContainer.isEmpty()); - keyPrefixesForContainer = containerDbServiceProvider + keyPrefixesForContainer = reconContainerMetadataManager .getKeyPrefixesForContainer(3); assertEquals(1, keyPrefixesForContainer.size()); - assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(1L)); - assertEquals(0, containerDbServiceProvider.getKeyCountForContainer(2L)); - assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(3L)); + assertEquals(1, reconContainerMetadataManager.getKeyCountForContainer(1L)); + assertEquals(0, reconContainerMetadataManager.getKeyCountForContainer(2L)); + assertEquals(1, reconContainerMetadataManager.getKeyCountForContainer(3L)); // Process PUT & DELETE event. containerKeyMapperTask.process(omUpdateEventBatch); - keyPrefixesForContainer = containerDbServiceProvider + keyPrefixesForContainer = reconContainerMetadataManager .getKeyPrefixesForContainer(1); assertEquals(1, keyPrefixesForContainer.size()); - keyPrefixesForContainer = containerDbServiceProvider + keyPrefixesForContainer = reconContainerMetadataManager .getKeyPrefixesForContainer(2); assertEquals(1, keyPrefixesForContainer.size()); - keyPrefixesForContainer = containerDbServiceProvider + keyPrefixesForContainer = reconContainerMetadataManager .getKeyPrefixesForContainer(3); assertTrue(keyPrefixesForContainer.isEmpty()); - assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(1L)); - assertEquals(1, containerDbServiceProvider.getKeyCountForContainer(2L)); - assertEquals(0, containerDbServiceProvider.getKeyCountForContainer(3L)); + assertEquals(1, reconContainerMetadataManager.getKeyCountForContainer(1L)); + assertEquals(1, reconContainerMetadataManager.getKeyCountForContainer(2L)); + assertEquals(0, reconContainerMetadataManager.getKeyCountForContainer(3L)); // Test if container count is updated - assertEquals(3, containerDbServiceProvider.getCountForContainers()); + assertEquals(3, reconContainerMetadataManager.getCountForContainers()); } private OmKeyInfo buildOmKeyInfo(String volume, From 17dd31d843344cb8381d5e5c687b53c8e83a4ee6 Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Thu, 24 Jun 2021 12:42:25 -0400 Subject: [PATCH 03/15] refactor ReconDBProvider to ReconRocksDB, added unit tests (cherry picked from commit 2b295247ee8fc03f13620e73adc8080a40872a00) --- .../ozone/recon/ReconControllerModule.java | 11 +- .../hadoop/ozone/recon/ReconServer.java | 18 ++- .../ozone/recon/api/types/NSSummary.java | 6 + .../ozone/recon/codec/NSSummaryCodec.java | 4 +- .../spi/ReconContainerMetadataManager.java | 5 - .../spi/ReconNSSummaryMetadataManager.java | 5 - .../spi/ReconNamespaceSummaryManager.java | 17 +++ .../ReconContainerMetadataManagerImpl.java | 40 ++----- .../ozone/recon/spi/impl/ReconDBProvider.java | 95 ---------------- .../ReconNSSummaryMetadataManagerImpl.java | 8 -- .../ReconNamespaceSummaryManagerImpl.java | 58 ++++++++++ .../ozone/recon/spi/impl/ReconRocksDB.java | 106 ++++++++++++++++-- .../hadoop/ozone/recon/ReconTestInjector.java | 9 +- .../TestReconNamespaceSummaryManagerImpl.java | 76 +++++++++++++ ...nDBProvider.java => TestReconRocksDB.java} | 14 +-- 15 files changed, 294 insertions(+), 178 deletions(-) delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNSSummaryMetadataManager.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java delete mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNSSummaryMetadataManagerImpl.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java create mode 100644 hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java rename hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/{TestReconDBProvider.java => TestReconRocksDB.java} (87%) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index c8435d330d11..1ac042df2ae0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -40,11 +40,9 @@ import org.apache.hadoop.ozone.recon.scm.ReconStorageContainerManagerFacade; import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerMetadataManagerImpl; -import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; -import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; -import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.*; import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask; import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; import org.apache.hadoop.ozone.recon.tasks.TableCountTask; @@ -85,8 +83,7 @@ public class ReconControllerModule extends AbstractModule { protected void configure() { bind(OzoneConfiguration.class).toProvider(ConfigurationProvider.class); bind(ReconHttpServer.class).in(Singleton.class); - bind(DBStore.class) - .toProvider(ReconDBProvider.class).in(Singleton.class); + bind(ReconRocksDB.class).in(Singleton.class); bind(ReconOMMetadataManager.class) .to(ReconOmMetadataManagerImpl.class); bind(OMMetadataManager.class).to(ReconOmMetadataManagerImpl.class); @@ -94,6 +91,8 @@ protected void configure() { bind(ContainerHealthSchemaManager.class).in(Singleton.class); bind(ReconContainerMetadataManager.class) .to(ReconContainerMetadataManagerImpl.class).in(Singleton.class); + bind(ReconNamespaceSummaryManager.class) + .to(ReconNamespaceSummaryManagerImpl.class).in(Singleton.class); bind(OzoneManagerServiceProvider.class) .to(OzoneManagerServiceProviderImpl.class).in(Singleton.class); bind(ReconUtils.class).in(Singleton.class); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index af90095e6a59..f13ca9f7539f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -31,7 +31,10 @@ import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; +import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; +import org.apache.hadoop.ozone.recon.spi.impl.ReconRocksDB; import org.apache.hadoop.ozone.util.OzoneVersionInfo; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -58,6 +61,8 @@ public class ReconServer extends GenericCli { private ReconHttpServer httpServer; private ReconContainerMetadataManager reconContainerMetadataManager; private OzoneManagerServiceProvider ozoneManagerServiceProvider; + private ReconRocksDB reconRocksDB; + private ReconNamespaceSummaryManager reconNamespaceSummaryManager; private OzoneStorageContainerManager reconStorageContainerManager; private OzoneConfiguration configuration; @@ -93,8 +98,12 @@ protected void configureServlets() { LOG.info("Initializing Recon server..."); try { loginReconUserIfSecurityEnabled(configuration); + this.reconRocksDB = injector.getInstance(ReconRocksDB.class); + LOG.info("Recon Rocks DB."); this.reconContainerMetadataManager = injector.getInstance(ReconContainerMetadataManager.class); + this.reconNamespaceSummaryManager = + injector.getInstance(ReconNamespaceSummaryManager.class); ReconSchemaManager reconSchemaManager = injector.getInstance(ReconSchemaManager.class); @@ -159,8 +168,8 @@ public void stop() throws Exception { if (ozoneManagerServiceProvider != null) { ozoneManagerServiceProvider.stop(); } - if (reconContainerMetadataManager != null) { - reconContainerMetadataManager.close(); + if (reconRocksDB != null) { + reconRocksDB.close(); } isStarted = false; } @@ -238,6 +247,11 @@ public ReconContainerMetadataManager getContainerDBServiceProvider() { return reconContainerMetadataManager; } + @VisibleForTesting + public ReconNamespaceSummaryManager getReconNamespaceSummaryManager() { + return reconNamespaceSummaryManager; + } + @VisibleForTesting ReconHttpServer getHttpServer() { return httpServer; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java index 4118063d7d5a..792a8c57cf43 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java @@ -20,6 +20,12 @@ public NSSummary() { this.fileSizeBucket = new int[ReconConstants.NUM_OF_BINS]; } + public NSSummary(int numOfFiles, int sizeOfFiles, int[] bucket) { + this.numOfFiles = numOfFiles; + this.sizeOfFiles = sizeOfFiles; + setFileSizeBucket(bucket); + } + public int getNumOfFiles() { return numOfFiles; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index 6a42eb882137..c6ba0a41dbc9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -31,11 +31,11 @@ public byte[] toPersistedFormat(NSSummary object) throws IOException { @Override public NSSummary fromPersistedFormat(byte[] rawData) throws IOException { - assert(rawData.length == NUM_OF_INTS); + assert(rawData.length == NUM_OF_INTS * Integer.BYTES); DataInputStream in = new DataInputStream(new ByteArrayInputStream(rawData)); NSSummary res = new NSSummary(); res.setNumOfFiles(in.readInt()); - res.setNumOfFiles(in.readInt()); + res.setSizeOfFiles(in.readInt()); int[] fileSizeBucket = new int[ReconConstants.NUM_OF_BINS]; for (int i = 0; i < ReconConstants.NUM_OF_BINS && in.available() > 0; ++i) { fileSizeBucket[i] = in.readInt(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java index 4f122b1041a5..075f6441b6c6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java @@ -34,11 +34,6 @@ @InterfaceStability.Unstable public interface ReconContainerMetadataManager { - /* - * Close the container DB - */ - void close() throws Exception; - /** * Create new container DB and bulk Store the container to Key prefix * mapping. diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNSSummaryMetadataManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNSSummaryMetadataManager.java deleted file mode 100644 index 184d78de4e40..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNSSummaryMetadataManager.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.apache.hadoop.ozone.recon.spi; - -public interface ReconNSSummaryMetadataManager { - -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java new file mode 100644 index 000000000000..6d4a437a3a14 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java @@ -0,0 +1,17 @@ +package org.apache.hadoop.ozone.recon.spi; + +import org.apache.hadoop.hdds.annotation.InterfaceStability; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; + +import java.io.IOException; + +@InterfaceStability.Unstable +public interface ReconNamespaceSummaryManager { + + // TODO: getDiskUsage? (w/w.o replication) + void initNSSummaryTable() throws IOException; + + void storeNSSummary(long objectId, NSSummary nsSummary) throws IOException; + + NSSummary getNSSummary(long objectId) throws IOException; +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java index c273c00ce94f..031d9d193df0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.recon.spi.impl; import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_COUNT_KEY; -import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider.getNewDBStore; +import static org.apache.hadoop.ozone.recon.spi.impl.ReconRocksDB.clearTable; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.CONTAINER_KEY; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.CONTAINER_KEY_COUNT; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.REPLICA_HISTORY; @@ -72,32 +72,19 @@ public class ReconContainerMetadataManagerImpl containerReplicaHistoryTable; private GlobalStatsDao globalStatsDao; - @Inject - private OzoneConfiguration configuration; - - @Inject private DBStore containerDbStore; @Inject private Configuration sqlConfiguration; @Inject - public ReconContainerMetadataManagerImpl(DBStore dbStore, + public ReconContainerMetadataManagerImpl(ReconRocksDB reconRocksDB, Configuration sqlConfiguration) { - containerDbStore = dbStore; + containerDbStore = reconRocksDB.getDbStore(); globalStatsDao = new GlobalStatsDao(sqlConfiguration); initializeTables(); } - @Override - public void close() throws Exception { - if (containerDbStore != null) { - LOG.info("Stopping ContainerKeyDB Service Provider"); - containerDbStore.close(); - containerDbStore = null; - } - } - /** * Initialize a new container DB instance, getting rid of the old instance * and then storing the passed in container prefix counts into the created @@ -110,25 +97,9 @@ public void close() throws Exception { public void initNewContainerDB(Map containerKeyPrefixCounts) throws IOException { - - File oldDBLocation = containerDbStore.getDbLocation(); - try { - containerDbStore.close(); - } catch (Exception e) { - LOG.warn("Unable to close old Recon container key DB at {}.", - containerDbStore.getDbLocation().getAbsolutePath()); - } - containerDbStore = getNewDBStore(configuration); - LOG.info("Creating new Recon Container DB at {}", - containerDbStore.getDbLocation().getAbsolutePath()); + // clear and re-init all container-related tables initializeTables(); - if (oldDBLocation.exists()) { - LOG.info("Cleaning up old Recon Container key DB at {}.", - oldDBLocation.getAbsolutePath()); - FileUtils.deleteDirectory(oldDBLocation); - } - if (containerKeyPrefixCounts != null) { for (Map.Entry entry : containerKeyPrefixCounts.entrySet()) { @@ -145,6 +116,9 @@ public void initNewContainerDB(Map */ private void initializeTables() { try { + clearTable(this.containerKeyTable); + clearTable(this.containerKeyCountTable); + clearTable(this.containerReplicaHistoryTable); this.containerKeyTable = CONTAINER_KEY.getTable(containerDbStore); this.containerKeyCountTable = CONTAINER_KEY_COUNT.getTable(containerDbStore); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java deleted file mode 100644 index 9242d75f7111..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.recon.spi.impl; - -import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB; -import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; - -import java.io.File; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.recon.ReconUtils; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.ProvisionException; - -/** - * Provider for the Recon container DB (Metadata store). - */ -public class ReconDBProvider implements Provider { - - @VisibleForTesting - private static final Logger LOG = - LoggerFactory.getLogger(ReconDBProvider.class); - - private OzoneConfiguration configuration; - private ReconUtils reconUtils; - - @Inject - public ReconDBProvider(OzoneConfiguration configuration, - ReconUtils reconUtils) { - this.configuration = configuration; - this.reconUtils = reconUtils; - } - - @Override - public DBStore get() { - DBStore dbStore; - File reconDbDir = - reconUtils.getReconDbDir(configuration, OZONE_RECON_DB_DIR); - File lastKnownContainerKeyDb = - reconUtils.getLastKnownDB(reconDbDir, RECON_CONTAINER_KEY_DB); - if (lastKnownContainerKeyDb != null) { - LOG.info("Last known container-key DB : {}", - lastKnownContainerKeyDb.getAbsolutePath()); - dbStore = initializeDBStore(configuration, - lastKnownContainerKeyDb.getName()); - } else { - dbStore = getNewDBStore(configuration); - } - if (dbStore == null) { - throw new ProvisionException("Unable to provide instance of DBStore " + - "store."); - } - return dbStore; - } - - private static DBStore initializeDBStore(OzoneConfiguration configuration, - String dbName) { - DBStore dbStore = null; - try { - dbStore = DBStoreBuilder.createDBStore(configuration, - new ReconDBDefinition(dbName)); - } catch (Exception ex) { - LOG.error("Unable to initialize Recon container metadata store.", ex); - } - return dbStore; - } - - static DBStore getNewDBStore(OzoneConfiguration configuration) { - String dbName = RECON_CONTAINER_KEY_DB + "_" + System.currentTimeMillis(); - return initializeDBStore(configuration, dbName); - } -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNSSummaryMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNSSummaryMetadataManagerImpl.java deleted file mode 100644 index cab88ef746ea..000000000000 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNSSummaryMetadataManagerImpl.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.apache.hadoop.ozone.recon.spi.impl; - -import org.apache.hadoop.ozone.recon.spi.ReconNSSummaryMetadataManager; - -public class ReconNSSummaryMetadataManagerImpl - implements ReconNSSummaryMetadataManager { - -} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java new file mode 100644 index 000000000000..19d886d23db1 --- /dev/null +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java @@ -0,0 +1,58 @@ +package org.apache.hadoop.ozone.recon.spi.impl; + +import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import static org.apache.hadoop.ozone.recon.spi.impl.ReconRocksDB.clearTable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.NAMESPACE_SUMMARY; + +import java.io.IOException; + +public class ReconNamespaceSummaryManagerImpl + implements ReconNamespaceSummaryManager { + + private static final Logger LOG = + LoggerFactory.getLogger(ReconNamespaceSummaryManagerImpl.class); + private Table nsSummaryTable; + private DBStore namespaceDbStore; + + // TODO: compute disk usage here? + @Inject + public ReconNamespaceSummaryManagerImpl(ReconRocksDB reconRocksDB) throws IOException { + namespaceDbStore = reconRocksDB.getDbStore(); + initializeTable(); + } + + @Override + public void initNSSummaryTable() throws IOException { + initializeTable(); + } + + @Override + public void storeNSSummary(long objectId, NSSummary nsSummary) throws IOException { + nsSummaryTable.put(objectId, nsSummary); + } + + @Override + public NSSummary getNSSummary(long objectId) throws IOException { + return nsSummaryTable.get(objectId); + } + + private void initializeTable() { + try { + clearTable(this.nsSummaryTable); + this.nsSummaryTable = NAMESPACE_SUMMARY.getTable(namespaceDbStore); + } catch (IOException e) { + LOG.error("cannot initialize table"); + } + } + + public Table getNSSummaryTable() { + return nsSummaryTable; + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java index a6e30db5e014..a8b0c1c619d1 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java @@ -1,35 +1,117 @@ package org.apache.hadoop.ozone.recon.spi.impl; +import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; +import java.io.File; +import java.io.IOException; import javax.inject.Inject; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBStore; +import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.recon.ReconUtils; +import com.google.inject.ProvisionException; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.recon.api.types.NSSummary; -import org.apache.hadoop.ozone.recon.scm.ContainerReplicaHistoryList; +import org.apache.hadoop.hdds.utils.db.Table.KeyValue; public class ReconRocksDB { + private OzoneConfiguration configuration; + private ReconUtils reconUtils; + private DBStore dbStore; + + @VisibleForTesting + private static final Logger LOG = + LoggerFactory.getLogger(ReconRocksDB.class); + @Inject - ReconRocksDB(DBStore dbStore) { + ReconRocksDB(OzoneConfiguration configuration, ReconUtils reconUtils) { + this.configuration = configuration; + this.reconUtils = reconUtils; + this.dbStore = provideReconDB(); + } + public DBStore provideReconDB() { + DBStore db; + File reconDbDir = + reconUtils.getReconDbDir(configuration, OZONE_RECON_DB_DIR); + File lastKnownContainerKeyDb = + reconUtils.getLastKnownDB(reconDbDir, RECON_CONTAINER_KEY_DB); + if (lastKnownContainerKeyDb != null) { + LOG.info("Last known DB : {}", + lastKnownContainerKeyDb.getAbsolutePath()); + db = initializeDBStore(configuration, + lastKnownContainerKeyDb.getName()); + } else { + db = getNewDBStore(configuration); + } + if (db == null) { + throw new ProvisionException("Unable to provide instance of DBStore " + + "store."); + } + return db; } - public Table getContainerKeyTable() { - return null; + public DBStore getDbStore() { + return dbStore; } - public Table getContainerKeyCountTable() { - return null; + public void reInit() throws IOException { + File oldDBLocation = dbStore.getDbLocation(); + try { + dbStore.close(); + } catch (Exception e) { + LOG.warn("Unable to close old Recon container key DB at {}.", + dbStore.getDbLocation().getAbsolutePath()); + } + dbStore = getNewDBStore(configuration); + LOG.info("Creating new Recon Container DB at {}", + dbStore.getDbLocation().getAbsolutePath()); + + if (oldDBLocation.exists()) { + LOG.info("Cleaning up old Recon Container key DB at {}.", + oldDBLocation.getAbsolutePath()); + FileUtils.deleteDirectory(oldDBLocation); + } } - public Table getContainerReplicaTable() { - return null; + static void clearTable(Table table) throws IOException { + if (table == null) { + return; + } + TableIterator> tableIterator = table.iterator(); + while (tableIterator.hasNext()) { + KeyValue entry = tableIterator.next(); + table.delete(entry.getKey()); + } } - public Table getNSSummaryTable() { - return null; + static DBStore getNewDBStore(OzoneConfiguration configuration) { + String dbName = RECON_CONTAINER_KEY_DB + "_" + System.currentTimeMillis(); + return initializeDBStore(configuration, dbName); } - public void close() { + private static DBStore initializeDBStore(OzoneConfiguration configuration, + String dbName) { + DBStore dbStore = null; + try { + dbStore = DBStoreBuilder.createDBStore(configuration, + new ReconDBDefinition(dbName)); + } catch (Exception ex) { + LOG.error("Unable to initialize Recon container metadata store.", ex); + } + return dbStore; + } + public void close() throws Exception { + if (this.dbStore != null) { + dbStore.close(); + dbStore = null; + } } } \ No newline at end of file diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java index ddbb17bcaced..a6c9e6fbaa29 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java @@ -40,9 +40,11 @@ import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerMetadataManagerImpl; -import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; +import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; +import org.apache.hadoop.ozone.recon.spi.impl.ReconRocksDB; import org.junit.Assert; import org.junit.rules.TemporaryFolder; @@ -174,8 +176,9 @@ protected void configure() { if (withContainerDB) { bind(ReconContainerMetadataManager.class) .to(ReconContainerMetadataManagerImpl.class).in(Singleton.class); - bind(DBStore.class).toProvider(ReconDBProvider.class). - in(Singleton.class); + bind(ReconNamespaceSummaryManager.class) + .to(ReconNamespaceSummaryManagerImpl.class).in(Singleton.class); + bind(ReconRocksDB.class).in(Singleton.class); } for (Map.Entry entry : diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java new file mode 100644 index 000000000000..5b46bd3faeb6 --- /dev/null +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java @@ -0,0 +1,76 @@ +package org.apache.hadoop.ozone.recon.spi.impl; + +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; +import org.junit.*; +import org.junit.rules.TemporaryFolder; + +import javax.validation.constraints.AssertFalse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +public class TestReconNamespaceSummaryManagerImpl { + @ClassRule + public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); + private static ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager; + private static int[] TEST_BUCKET; + + @BeforeClass + public static void setupOnce() throws Exception { + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(TEMP_FOLDER) + .withReconSqlDb() + .withContainerDB() + .build(); + reconNamespaceSummaryManager = + reconTestInjector.getInstance(ReconNamespaceSummaryManagerImpl.class); + TEST_BUCKET = new int[40]; + for (int i = 0; i < 40; ++i) { + TEST_BUCKET[i] = i + 1; + } + } + + @Before + public void setUp() throws Exception { + // Clear namespace table before running each test + reconNamespaceSummaryManager.initNSSummaryTable(); + } + + @Test + public void testStoreAndGet() throws Exception { + putThreeNSMetadata(); + NSSummary summary = reconNamespaceSummaryManager.getNSSummary(1L); + NSSummary summary2 = reconNamespaceSummaryManager.getNSSummary(2L); + NSSummary summary3 = reconNamespaceSummaryManager.getNSSummary(3L); + Assert.assertEquals(1, summary.getNumOfFiles()); + Assert.assertEquals(2, summary.getSizeOfFiles()); + Assert.assertEquals(3, summary2.getNumOfFiles()); + Assert.assertEquals(4, summary2.getSizeOfFiles()); + Assert.assertEquals(5, summary3.getNumOfFiles()); + Assert.assertEquals(6, summary3.getSizeOfFiles()); + // non-existent key + Assert.assertNull(reconNamespaceSummaryManager.getNSSummary(0L)); + } + + @Test + public void testInitNSSummaryTable() throws IOException { + putThreeNSMetadata(); + Assert.assertFalse(reconNamespaceSummaryManager.getNSSummaryTable().isEmpty()); + reconNamespaceSummaryManager.initNSSummaryTable(); + Assert.assertTrue(reconNamespaceSummaryManager.getNSSummaryTable().isEmpty()); + } + + private void putThreeNSMetadata() throws IOException { + HashMap hmap = new HashMap<>(); + hmap.put(1L, new NSSummary(1, 2, TEST_BUCKET)); + hmap.put(2L, new NSSummary(3, 4, TEST_BUCKET)); + hmap.put(3L, new NSSummary(5, 6, TEST_BUCKET)); + for (Map.Entry entry: hmap.entrySet()) { + reconNamespaceSummaryManager.storeNSSummary( + (long)entry.getKey(), (NSSummary)entry.getValue()); + } + } +} diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java similarity index 87% rename from hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java rename to hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java index 5cdc66508a3b..07e01546dfa0 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.recon.spi.impl; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import java.io.File; @@ -26,6 +27,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBStore; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -40,7 +42,7 @@ * Tests the class that provides the instance of the DB Store used by Recon to * store its container - key data. */ -public class TestReconDBProvider { +public class TestReconRocksDB { @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); @@ -57,18 +59,16 @@ protected void configure() { OzoneConfiguration configuration = new OzoneConfiguration(); configuration.set(OZONE_RECON_DB_DIR, dbDir.getAbsolutePath()); bind(OzoneConfiguration.class).toInstance(configuration); - bind(DBStore.class).toProvider(ReconDBProvider.class).in( - Singleton.class); + bind(ReconRocksDB.class).in(Singleton.class); } }); } @Test public void testGet() throws Exception { - ReconDBProvider reconDBProvider = injector.getInstance( - ReconDBProvider.class); - DBStore dbStore = reconDBProvider.get(); - assertNotNull(dbStore); + ReconRocksDB reconRocksDB = injector.getInstance( + ReconRocksDB.class); + assertNotNull(reconRocksDB.getDbStore()); } } \ No newline at end of file From 62446f33e85b3c3805ea99733dc19f45e755950b Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Thu, 24 Jun 2021 13:12:19 -0400 Subject: [PATCH 04/15] checkstyle fix (cherry picked from commit 64fee94acf6dbbea39451c5ad85b7bbeea9c138d) --- .../ozone/recon/ReconControllerModule.java | 1 - .../hadoop/ozone/recon/ReconServer.java | 1 - .../ozone/recon/api/ContainerEndpoint.java | 3 +- .../ozone/recon/api/types/NSSummary.java | 5 ++-- .../ozone/recon/codec/NSSummaryCodec.java | 3 ++ .../spi/ReconNamespaceSummaryManager.java | 3 ++ .../ReconContainerMetadataManagerImpl.java | 3 -- .../ReconNamespaceSummaryManagerImpl.java | 8 ++++-- .../ozone/recon/spi/impl/ReconRocksDB.java | 8 ++++-- .../recon/tasks/ContainerKeyMapperTask.java | 4 +-- .../hadoop/ozone/recon/ReconTestInjector.java | 7 +++-- ...TestReconContainerMetadataManagerImpl.java | 3 +- .../TestReconNamespaceSummaryManagerImpl.java | 28 ++++++++++--------- .../recon/spi/impl/TestReconRocksDB.java | 3 -- 14 files changed, 45 insertions(+), 35 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index 1ac042df2ae0..3f293550de5a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.protocolPB.OmTransport; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index f13ca9f7539f..87693422c71a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -33,7 +33,6 @@ import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; import org.apache.hadoop.ozone.recon.spi.impl.ReconRocksDB; import org.apache.hadoop.ozone.util.OzoneVersionInfo; import org.apache.hadoop.security.SecurityUtil; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java index 8eb5ea807fac..8f3ad94ae35f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerEndpoint.java @@ -113,7 +113,8 @@ public Response getContainers( Map containersMap; long containersCount; try { - containersMap = reconContainerMetadataManager.getContainers(limit, prevKey); + containersMap = + reconContainerMetadataManager.getContainers(limit, prevKey); containersCount = reconContainerMetadataManager.getCountForContainers(); } catch (IOException ioEx) { throw new WebApplicationException(ioEx, diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java index 792a8c57cf43..9a60a9a8e79e 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java @@ -5,7 +5,7 @@ import java.util.Arrays; /** - * Class to encapsulate namespace metadata summaries from OM + * Class to encapsulate namespace metadata summaries from OM. */ public class NSSummary { @@ -16,7 +16,8 @@ public class NSSummary { public NSSummary() { this.numOfFiles = 0; this.sizeOfFiles = 0; - // TODO: I read the min is 1024(2^10), max is 1PB(2^50), so the number of buckets should be 40? + // TODO: I read the min is 1024(2^10), max is 1PB(2^50), + // so the number of buckets should be 40? this.fileSizeBucket = new int[ReconConstants.NUM_OF_BINS]; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index c6ba0a41dbc9..6870b9c3b11a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -10,6 +10,9 @@ import java.io.DataInputStream; import java.io.IOException; +/** + * Codec for Namespace Summary. + */ public class NSSummaryCodec implements Codec{ private final Codec integerCodec = new IntegerCodec(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java index 6d4a437a3a14..0e8ea61ced90 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java @@ -5,6 +5,9 @@ import java.io.IOException; +/** + * Interface for DB operations on NS summary. + */ @InterfaceStability.Unstable public interface ReconNamespaceSummaryManager { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java index 031d9d193df0..1b77c1c8bae9 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java @@ -24,7 +24,6 @@ import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.CONTAINER_KEY_COUNT; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.REPLICA_HISTORY; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -36,9 +35,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.recon.ReconUtils; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java index 19d886d23db1..4c188dea284b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java @@ -13,6 +13,9 @@ import java.io.IOException; +/** + * Wrapper functions for DB operations on recon namespace summary metadata. + */ public class ReconNamespaceSummaryManagerImpl implements ReconNamespaceSummaryManager { @@ -23,7 +26,7 @@ public class ReconNamespaceSummaryManagerImpl // TODO: compute disk usage here? @Inject - public ReconNamespaceSummaryManagerImpl(ReconRocksDB reconRocksDB) throws IOException { + public ReconNamespaceSummaryManagerImpl(ReconRocksDB reconRocksDB) { namespaceDbStore = reconRocksDB.getDbStore(); initializeTable(); } @@ -34,7 +37,8 @@ public void initNSSummaryTable() throws IOException { } @Override - public void storeNSSummary(long objectId, NSSummary nsSummary) throws IOException { + public void storeNSSummary(long objectId, NSSummary nsSummary) + throws IOException { nsSummaryTable.put(objectId, nsSummary); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java index a8b0c1c619d1..be5bc4f75791 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java @@ -13,13 +13,14 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.recon.ReconUtils; import com.google.inject.ProvisionException; -import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.Table.KeyValue; - +/** + * Provider for Recon's RDB. + */ public class ReconRocksDB { private OzoneConfiguration configuration; private ReconUtils reconUtils; @@ -84,7 +85,8 @@ static void clearTable(Table table) throws IOException { if (table == null) { return; } - TableIterator> tableIterator = table.iterator(); + TableIterator> + tableIterator = table.iterator(); while (tableIterator.hasNext()) { KeyValue entry = tableIterator.next(); table.delete(entry.getKey()); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java index 85e29cd1d74b..0e47d6d2ccb3 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java @@ -230,8 +230,8 @@ private void writeOMKeyToContainerDB(String key, OmKeyInfo omKeyInfo) } // update the count of keys for the given containerID - long keyCount = - reconContainerMetadataManager.getKeyCountForContainer(containerId); + long keyCount = reconContainerMetadataManager + .getKeyCountForContainer(containerId); // increment the count and update containerKeyCount. // keyCount will be 0 if containerID is not found. So, there is no diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java index a6c9e6fbaa29..e2ee1bdcb8b5 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java @@ -36,7 +36,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; -import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.ozone.recon.persistence.AbstractReconSqlDBTest; import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; @@ -175,9 +174,11 @@ protected void configure() { if (withContainerDB) { bind(ReconContainerMetadataManager.class) - .to(ReconContainerMetadataManagerImpl.class).in(Singleton.class); + .to(ReconContainerMetadataManagerImpl.class) + .in(Singleton.class); bind(ReconNamespaceSummaryManager.class) - .to(ReconNamespaceSummaryManagerImpl.class).in(Singleton.class); + .to(ReconNamespaceSummaryManagerImpl.class) + .in(Singleton.class); bind(ReconRocksDB.class).in(Singleton.class); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java index 6accb23154f0..66cdadca7ed3 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java @@ -208,7 +208,8 @@ public void testDoesContainerExists() throws Exception { reconContainerMetadataManager.storeContainerKeyCount(nextContainerId, 3L); assertTrue(reconContainerMetadataManager.doesContainerExists(containerId)); - assertTrue(reconContainerMetadataManager.doesContainerExists(nextContainerId)); + assertTrue(reconContainerMetadataManager. + doesContainerExists(nextContainerId)); assertFalse(reconContainerMetadataManager.doesContainerExists(0L)); assertFalse(reconContainerMetadataManager.doesContainerExists(3L)); } diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java index 5b46bd3faeb6..ee17c9ea3a8b 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java @@ -2,21 +2,21 @@ import org.apache.hadoop.ozone.recon.ReconTestInjector; import org.apache.hadoop.ozone.recon.api.types.NSSummary; -import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager; import org.junit.*; import org.junit.rules.TemporaryFolder; -import javax.validation.constraints.AssertFalse; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +/** + * Test for NSSummary manager. + */ public class TestReconNamespaceSummaryManagerImpl { @ClassRule public static final TemporaryFolder TEMP_FOLDER = new TemporaryFolder(); private static ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager; - private static int[] TEST_BUCKET; + private static int[] testBucket; @BeforeClass public static void setupOnce() throws Exception { @@ -25,11 +25,11 @@ public static void setupOnce() throws Exception { .withReconSqlDb() .withContainerDB() .build(); - reconNamespaceSummaryManager = - reconTestInjector.getInstance(ReconNamespaceSummaryManagerImpl.class); - TEST_BUCKET = new int[40]; + reconNamespaceSummaryManager = reconTestInjector.getInstance( + ReconNamespaceSummaryManagerImpl.class); + testBucket = new int[40]; for (int i = 0; i < 40; ++i) { - TEST_BUCKET[i] = i + 1; + testBucket[i] = i + 1; } } @@ -58,16 +58,18 @@ public void testStoreAndGet() throws Exception { @Test public void testInitNSSummaryTable() throws IOException { putThreeNSMetadata(); - Assert.assertFalse(reconNamespaceSummaryManager.getNSSummaryTable().isEmpty()); + Assert.assertFalse( + reconNamespaceSummaryManager.getNSSummaryTable().isEmpty()); reconNamespaceSummaryManager.initNSSummaryTable(); - Assert.assertTrue(reconNamespaceSummaryManager.getNSSummaryTable().isEmpty()); + Assert.assertTrue( + reconNamespaceSummaryManager.getNSSummaryTable().isEmpty()); } private void putThreeNSMetadata() throws IOException { HashMap hmap = new HashMap<>(); - hmap.put(1L, new NSSummary(1, 2, TEST_BUCKET)); - hmap.put(2L, new NSSummary(3, 4, TEST_BUCKET)); - hmap.put(3L, new NSSummary(5, 6, TEST_BUCKET)); + hmap.put(1L, new NSSummary(1, 2, testBucket)); + hmap.put(2L, new NSSummary(3, 4, testBucket)); + hmap.put(3L, new NSSummary(5, 6, testBucket)); for (Map.Entry entry: hmap.entrySet()) { reconNamespaceSummaryManager.storeNSSummary( (long)entry.getKey(), (NSSummary)entry.getValue()); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java index 07e01546dfa0..1cf0bd9898fe 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java @@ -19,15 +19,12 @@ package org.apache.hadoop.ozone.recon.spi.impl; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import java.io.File; import java.io.IOException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.utils.db.DBStore; -import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; From 63d354466b23c0f37c758751c806626fa413b07f Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Thu, 24 Jun 2021 14:38:34 -0400 Subject: [PATCH 05/15] fix license headers --- .../ozone/recon/api/types/NSSummary.java | 18 ++++++++++++++++++ .../ozone/recon/codec/NSSummaryCodec.java | 18 ++++++++++++++++++ .../spi/ReconNamespaceSummaryManager.java | 18 ++++++++++++++++++ .../impl/ReconNamespaceSummaryManagerImpl.java | 18 ++++++++++++++++++ .../ozone/recon/spi/impl/ReconRocksDB.java | 18 ++++++++++++++++++ .../TestReconNamespaceSummaryManagerImpl.java | 18 ++++++++++++++++++ 6 files changed, 108 insertions(+) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java index 9a60a9a8e79e..c6dbbb02e508 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.recon.api.types; import org.apache.hadoop.ozone.recon.ReconConstants; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index 6870b9c3b11a..b70f72747397 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.recon.codec; import org.apache.hadoop.hdds.utils.db.IntegerCodec; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java index 0e8ea61ced90..2bb395428707 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.recon.spi; import org.apache.hadoop.hdds.annotation.InterfaceStability; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java index 4c188dea284b..b4d20295ceef 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.recon.spi.impl; import org.apache.hadoop.hdds.utils.db.DBStore; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java index be5bc4f75791..4ac78f1860e2 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.recon.spi.impl; import static org.apache.hadoop.ozone.recon.ReconConstants.RECON_CONTAINER_KEY_DB; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java index ee17c9ea3a8b..3aff907dd40c 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconNamespaceSummaryManagerImpl.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.ozone.recon.spi.impl; import org.apache.hadoop.ozone.recon.ReconTestInjector; From f0cbfbb264c8c52d2a5f572814f446df6d6b80c2 Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Thu, 24 Jun 2021 23:03:25 -0400 Subject: [PATCH 06/15] renaming and nit --- .../recon/TestReconWithOzoneManagerHA.java | 2 +- .../ozone/recon/ReconControllerModule.java | 8 +++-- .../hadoop/ozone/recon/ReconServer.java | 18 ++++------- .../ozone/recon/api/types/NSSummary.java | 4 +-- .../ozone/recon/codec/NSSummaryCodec.java | 2 +- .../spi/ReconContainerMetadataManager.java | 2 +- .../spi/ReconNamespaceSummaryManager.java | 3 +- .../ReconContainerMetadataManagerImpl.java | 13 ++++---- .../recon/spi/impl/ReconDBDefinition.java | 6 ++-- ...ReconRocksDB.java => ReconDBProvider.java} | 32 ++++--------------- .../ReconNamespaceSummaryManagerImpl.java | 13 ++++---- .../recon/tasks/ContainerKeyMapperTask.java | 3 +- .../hadoop/ozone/recon/ReconTestInjector.java | 4 +-- .../hadoop/ozone/recon/api/TestEndpoints.java | 1 + ...TestReconContainerMetadataManagerImpl.java | 5 +-- ...nRocksDB.java => TestReconDBProvider.java} | 10 +++--- 16 files changed, 53 insertions(+), 73 deletions(-) rename hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/{ReconRocksDB.java => ReconDBProvider.java} (79%) rename hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/{TestReconRocksDB.java => TestReconDBProvider.java} (89%) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java index fc89af3bfe0e..3903275384ce 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java @@ -131,7 +131,7 @@ public void testReconGetsSnapshotFromLeader() throws Exception { impl.syncDataFromOM(); ReconContainerMetadataManager reconContainerMetadataManager = - cluster.getReconServer().getContainerDBServiceProvider(); + cluster.getReconServer().getRCMM(); TableIterator iterator = reconContainerMetadataManager.getContainerTableIterator(); String reconKeyPrefix = null; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index 3f293550de5a..27c407e7d1da 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -41,7 +41,11 @@ import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.*; +import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerMetadataManagerImpl; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; +import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; +import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; +import org.apache.hadoop.ozone.recon.spi.impl.StorageContainerServiceProviderImpl; import org.apache.hadoop.ozone.recon.tasks.ContainerKeyMapperTask; import org.apache.hadoop.ozone.recon.tasks.FileSizeCountTask; import org.apache.hadoop.ozone.recon.tasks.TableCountTask; @@ -82,7 +86,7 @@ public class ReconControllerModule extends AbstractModule { protected void configure() { bind(OzoneConfiguration.class).toProvider(ConfigurationProvider.class); bind(ReconHttpServer.class).in(Singleton.class); - bind(ReconRocksDB.class).in(Singleton.class); + bind(ReconDBProvider.class).in(Singleton.class); bind(ReconOMMetadataManager.class) .to(ReconOmMetadataManagerImpl.class); bind(OMMetadataManager.class).to(ReconOmMetadataManagerImpl.class); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index 87693422c71a..d2cde334b451 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -33,7 +33,7 @@ import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider; -import org.apache.hadoop.ozone.recon.spi.impl.ReconRocksDB; +import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; import org.apache.hadoop.ozone.util.OzoneVersionInfo; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -60,7 +60,7 @@ public class ReconServer extends GenericCli { private ReconHttpServer httpServer; private ReconContainerMetadataManager reconContainerMetadataManager; private OzoneManagerServiceProvider ozoneManagerServiceProvider; - private ReconRocksDB reconRocksDB; + private ReconDBProvider reconDBProvider; private ReconNamespaceSummaryManager reconNamespaceSummaryManager; private OzoneStorageContainerManager reconStorageContainerManager; private OzoneConfiguration configuration; @@ -97,8 +97,7 @@ protected void configureServlets() { LOG.info("Initializing Recon server..."); try { loginReconUserIfSecurityEnabled(configuration); - this.reconRocksDB = injector.getInstance(ReconRocksDB.class); - LOG.info("Recon Rocks DB."); + this.reconDBProvider = injector.getInstance(ReconDBProvider.class); this.reconContainerMetadataManager = injector.getInstance(ReconContainerMetadataManager.class); this.reconNamespaceSummaryManager = @@ -167,8 +166,8 @@ public void stop() throws Exception { if (ozoneManagerServiceProvider != null) { ozoneManagerServiceProvider.stop(); } - if (reconRocksDB != null) { - reconRocksDB.close(); + if (reconDBProvider != null) { + reconDBProvider.close(); } isStarted = false; } @@ -242,15 +241,10 @@ public StorageContainerServiceProvider getStorageContainerServiceProvider() { } @VisibleForTesting - public ReconContainerMetadataManager getContainerDBServiceProvider() { + public ReconContainerMetadataManager getRCMM() { return reconContainerMetadataManager; } - @VisibleForTesting - public ReconNamespaceSummaryManager getReconNamespaceSummaryManager() { - return reconNamespaceSummaryManager; - } - @VisibleForTesting ReconHttpServer getHttpServer() { return httpServer; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java index c6dbbb02e508..86eabd3f1f20 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/NSSummary.java @@ -34,8 +34,6 @@ public class NSSummary { public NSSummary() { this.numOfFiles = 0; this.sizeOfFiles = 0; - // TODO: I read the min is 1024(2^10), max is 1PB(2^50), - // so the number of buckets should be 40? this.fileSizeBucket = new int[ReconConstants.NUM_OF_BINS]; } @@ -54,7 +52,7 @@ public int getSizeOfFiles() { } public int[] getFileSizeBucket() { - return fileSizeBucket; + return Arrays.copyOf(this.fileSizeBucket, ReconConstants.NUM_OF_BINS); } public void setNumOfFiles(int numOfFiles) { diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index b70f72747397..bb8bdcc11ca6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -31,7 +31,7 @@ /** * Codec for Namespace Summary. */ -public class NSSummaryCodec implements Codec{ +public class NSSummaryCodec implements Codec { private final Codec integerCodec = new IntegerCodec(); // 2 int fields + 40-length int array diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java index 075f6441b6c6..2c27e7190fac 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java @@ -40,7 +40,7 @@ public interface ReconContainerMetadataManager { * @param containerKeyPrefixCounts Map of containerId, key-prefix tuple to * key count. */ - void initNewContainerDB(Map + void reinitWithNewContainerDataFromOm(Map containerKeyPrefixCounts) throws IOException; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java index 2bb395428707..5264824b4785 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java @@ -24,12 +24,11 @@ import java.io.IOException; /** - * Interface for DB operations on NS summary. + * Interface for DB operations on NSSummary. */ @InterfaceStability.Unstable public interface ReconNamespaceSummaryManager { - // TODO: getDiskUsage? (w/w.o replication) void initNSSummaryTable() throws IOException; void storeNSSummary(long objectId, NSSummary nsSummary) throws IOException; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java index 1b77c1c8bae9..d93ccad7e4ab 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java @@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.recon.spi.impl; import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_COUNT_KEY; -import static org.apache.hadoop.ozone.recon.spi.impl.ReconRocksDB.clearTable; +import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider.truncateTable; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.CONTAINER_KEY; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.CONTAINER_KEY_COUNT; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.REPLICA_HISTORY; @@ -75,9 +75,9 @@ public class ReconContainerMetadataManagerImpl private Configuration sqlConfiguration; @Inject - public ReconContainerMetadataManagerImpl(ReconRocksDB reconRocksDB, + public ReconContainerMetadataManagerImpl(ReconDBProvider reconDBProvider, Configuration sqlConfiguration) { - containerDbStore = reconRocksDB.getDbStore(); + containerDbStore = reconDBProvider.getDbStore(); globalStatsDao = new GlobalStatsDao(sqlConfiguration); initializeTables(); } @@ -91,10 +91,12 @@ public ReconContainerMetadataManagerImpl(ReconRocksDB reconRocksDB, * @throws IOException */ @Override - public void initNewContainerDB(Map + public void reinitWithNewContainerDataFromOm(Map containerKeyPrefixCounts) throws IOException { // clear and re-init all container-related tables + truncateTable(this.containerKeyTable); + truncateTable(this.containerKeyCountTable); initializeTables(); if (containerKeyPrefixCounts != null) { @@ -113,9 +115,6 @@ public void initNewContainerDB(Map */ private void initializeTables() { try { - clearTable(this.containerKeyTable); - clearTable(this.containerKeyCountTable); - clearTable(this.containerReplicaHistoryTable); this.containerKeyTable = CONTAINER_KEY.getTable(containerDbStore); this.containerKeyCountTable = CONTAINER_KEY_COUNT.getTable(containerDbStore); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java index ed1ed5fd7d58..cfaa2b050827 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java @@ -17,8 +17,10 @@ * */ package org.apache.hadoop.ozone.recon.spi.impl; - -import org.apache.hadoop.hdds.utils.db.*; +import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; +import org.apache.hadoop.hdds.utils.db.DBDefinition; +import org.apache.hadoop.hdds.utils.db.IntegerCodec; +import org.apache.hadoop.hdds.utils.db.LongCodec; import org.apache.hadoop.ozone.recon.ReconServerConfigKeys; import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix; import org.apache.hadoop.ozone.recon.codec.ContainerReplicaHistoryListCodec; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java similarity index 79% rename from hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java rename to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java index 4ac78f1860e2..fe4b3ee13ac6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconRocksDB.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java @@ -24,7 +24,6 @@ import javax.inject.Inject; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; @@ -39,17 +38,17 @@ /** * Provider for Recon's RDB. */ -public class ReconRocksDB { +public class ReconDBProvider { private OzoneConfiguration configuration; private ReconUtils reconUtils; private DBStore dbStore; @VisibleForTesting private static final Logger LOG = - LoggerFactory.getLogger(ReconRocksDB.class); + LoggerFactory.getLogger(ReconDBProvider.class); @Inject - ReconRocksDB(OzoneConfiguration configuration, ReconUtils reconUtils) { + ReconDBProvider(OzoneConfiguration configuration, ReconUtils reconUtils) { this.configuration = configuration; this.reconUtils = reconUtils; this.dbStore = provideReconDB(); @@ -62,7 +61,7 @@ public DBStore provideReconDB() { File lastKnownContainerKeyDb = reconUtils.getLastKnownDB(reconDbDir, RECON_CONTAINER_KEY_DB); if (lastKnownContainerKeyDb != null) { - LOG.info("Last known DB : {}", + LOG.info("Last known Recon DB : {}", lastKnownContainerKeyDb.getAbsolutePath()); db = initializeDBStore(configuration, lastKnownContainerKeyDb.getName()); @@ -80,26 +79,7 @@ public DBStore getDbStore() { return dbStore; } - public void reInit() throws IOException { - File oldDBLocation = dbStore.getDbLocation(); - try { - dbStore.close(); - } catch (Exception e) { - LOG.warn("Unable to close old Recon container key DB at {}.", - dbStore.getDbLocation().getAbsolutePath()); - } - dbStore = getNewDBStore(configuration); - LOG.info("Creating new Recon Container DB at {}", - dbStore.getDbLocation().getAbsolutePath()); - - if (oldDBLocation.exists()) { - LOG.info("Cleaning up old Recon Container key DB at {}.", - oldDBLocation.getAbsolutePath()); - FileUtils.deleteDirectory(oldDBLocation); - } - } - - static void clearTable(Table table) throws IOException { + static void truncateTable(Table table) throws IOException { if (table == null) { return; } @@ -134,4 +114,4 @@ public void close() throws Exception { dbStore = null; } } -} \ No newline at end of file +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java index b4d20295ceef..9283bd3dea5b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.ozone.recon.api.types.NSSummary; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; -import static org.apache.hadoop.ozone.recon.spi.impl.ReconRocksDB.clearTable; +import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider.truncateTable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,10 +42,9 @@ public class ReconNamespaceSummaryManagerImpl private Table nsSummaryTable; private DBStore namespaceDbStore; - // TODO: compute disk usage here? @Inject - public ReconNamespaceSummaryManagerImpl(ReconRocksDB reconRocksDB) { - namespaceDbStore = reconRocksDB.getDbStore(); + public ReconNamespaceSummaryManagerImpl(ReconDBProvider reconDBProvider) { + namespaceDbStore = reconDBProvider.getDbStore(); initializeTable(); } @@ -67,8 +66,10 @@ public NSSummary getNSSummary(long objectId) throws IOException { private void initializeTable() { try { - clearTable(this.nsSummaryTable); - this.nsSummaryTable = NAMESPACE_SUMMARY.getTable(namespaceDbStore); + truncateTable(this.nsSummaryTable); + if (nsSummaryTable == null) { + this.nsSummaryTable = NAMESPACE_SUMMARY.getTable(namespaceDbStore); + } } catch (IOException e) { LOG.error("cannot initialize table"); } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java index 0e47d6d2ccb3..544b0db1713b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java @@ -74,7 +74,8 @@ public Pair reprocess(OMMetadataManager omMetadataManager) { Instant start = Instant.now(); // initialize new container DB - reconContainerMetadataManager.initNewContainerDB(new HashMap<>()); + reconContainerMetadataManager + .reinitWithNewContainerDataFromOm(new HashMap<>()); Table omKeyInfoTable = omMetadataManager.getKeyTable(); try (TableIterator> diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java index e2ee1bdcb8b5..e4484cd5a015 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/ReconTestInjector.java @@ -43,7 +43,7 @@ import org.apache.hadoop.ozone.recon.spi.OzoneManagerServiceProvider; import org.apache.hadoop.ozone.recon.spi.impl.ReconContainerMetadataManagerImpl; import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl; -import org.apache.hadoop.ozone.recon.spi.impl.ReconRocksDB; +import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider; import org.junit.Assert; import org.junit.rules.TemporaryFolder; @@ -179,7 +179,7 @@ protected void configure() { bind(ReconNamespaceSummaryManager.class) .to(ReconNamespaceSummaryManagerImpl.class) .in(Singleton.class); - bind(ReconRocksDB.class).in(Singleton.class); + bind(ReconDBProvider.class).in(Singleton.class); } for (Map.Entry entry : diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java index 7179865897f8..03aa184b5f74 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestEndpoints.java @@ -458,6 +458,7 @@ public void testGetDatanodes() throws Exception { .findFirst().orElse(null); return (datanodeMetadata1 != null && datanodeMetadata1.getContainers() == 1 && + datanodeMetadata1.getOpenContainers() == 1 && reconScm.getPipelineManager() .getContainersInPipeline(pipeline.getId()).size() == 1); }); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java index 66cdadca7ed3..245cb03d6730 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconContainerMetadataManagerImpl.java @@ -64,7 +64,7 @@ public static void setupOnce() throws Exception { @Before public void setUp() throws Exception { // Reset containerDB before running each test - reconContainerMetadataManager.initNewContainerDB(null); + reconContainerMetadataManager.reinitWithNewContainerDataFromOm(null); } private void populateKeysInContainers(long containerId1, long containerId2) @@ -123,7 +123,8 @@ public void testInitNewContainerDB() throws Exception { "V1/B3/K2", 0); prefixCounts.put(ckp5, 15); - reconContainerMetadataManager.initNewContainerDB(prefixCounts); + reconContainerMetadataManager + .reinitWithNewContainerDataFromOm(prefixCounts); Map keyPrefixesForContainer = reconContainerMetadataManager.getKeyPrefixesForContainer(containerId); diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java similarity index 89% rename from hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java rename to hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java index 1cf0bd9898fe..3366843e135f 100644 --- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconRocksDB.java +++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/spi/impl/TestReconDBProvider.java @@ -39,7 +39,7 @@ * Tests the class that provides the instance of the DB Store used by Recon to * store its container - key data. */ -public class TestReconRocksDB { +public class TestReconDBProvider { @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); @@ -56,16 +56,16 @@ protected void configure() { OzoneConfiguration configuration = new OzoneConfiguration(); configuration.set(OZONE_RECON_DB_DIR, dbDir.getAbsolutePath()); bind(OzoneConfiguration.class).toInstance(configuration); - bind(ReconRocksDB.class).in(Singleton.class); + bind(ReconDBProvider.class).in(Singleton.class); } }); } @Test public void testGet() throws Exception { - ReconRocksDB reconRocksDB = injector.getInstance( - ReconRocksDB.class); - assertNotNull(reconRocksDB.getDbStore()); + ReconDBProvider reconDBProvider = injector.getInstance( + ReconDBProvider.class); + assertNotNull(reconDBProvider.getDbStore()); } } \ No newline at end of file From bfa2f94a0d28d8590a62edb4779cd35ad2429eba Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Fri, 25 Jun 2021 09:55:38 -0400 Subject: [PATCH 07/15] use file size max upper bound to calculate num of bins, not hardcoded --- .../java/org/apache/hadoop/ozone/recon/ReconConstants.java | 6 +++++- .../apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index b4cfcf291e76..d02a2c1bf90f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -58,5 +58,9 @@ private ReconConstants() { + PIPELINE_DB_SUFFIX; public static final String RECON_SCM_NODE_DB = "recon-node.db"; - public static final int NUM_OF_BINS = 40; + public static final long MAX_FILE_SIZE_UPPER_BOUND = 1125899906842624L; + public static final long MIN_FILE_SIZE_UPPER_BOUND = 1024L; + public static final int NUM_OF_BINS = (int)(Math.log( + MAX_FILE_SIZE_UPPER_BOUND) / Math.log(2) - Math.log( + MIN_FILE_SIZE_UPPER_BOUND) / Math.log(2)) + 1; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index e14096a86de5..f4cd6561b7f0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.recon.ReconConstants; import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition; import org.hadoop.ozone.recon.schema.tables.daos.FileCountBySizeDao; import org.hadoop.ozone.recon.schema.tables.pojos.FileCountBySize; @@ -53,7 +54,6 @@ public class FileSizeCountTask implements ReconOmTask { LoggerFactory.getLogger(FileSizeCountTask.class); // 1125899906842624L = 1PB - private static final long MAX_FILE_SIZE_UPPER_BOUND = 1125899906842624L; private FileCountBySizeDao fileCountBySizeDao; private DSLContext dslContext; @@ -167,7 +167,7 @@ public Pair process(OMUpdateEventBatch events) { } private long getFileSizeUpperBound(long fileSize) { - if (fileSize >= MAX_FILE_SIZE_UPPER_BOUND) { + if (fileSize >= ReconConstants.MAX_FILE_SIZE_UPPER_BOUND) { return Long.MAX_VALUE; } int index = nextClosestPowerIndexOfTwo(fileSize); From 33629ee78b88b442ba794e9f887c63fd104a3b37 Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Fri, 25 Jun 2021 13:32:36 -0400 Subject: [PATCH 08/15] nit on init & truncate table --- .../ReconNamespaceSummaryManagerImpl.java | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java index 9283bd3dea5b..01f8c4e3e4a6 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java @@ -23,8 +23,6 @@ import org.apache.hadoop.ozone.recon.api.types.NSSummary; import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider.truncateTable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.inject.Inject; import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.NAMESPACE_SUMMARY; @@ -37,20 +35,19 @@ public class ReconNamespaceSummaryManagerImpl implements ReconNamespaceSummaryManager { - private static final Logger LOG = - LoggerFactory.getLogger(ReconNamespaceSummaryManagerImpl.class); private Table nsSummaryTable; private DBStore namespaceDbStore; @Inject - public ReconNamespaceSummaryManagerImpl(ReconDBProvider reconDBProvider) { + public ReconNamespaceSummaryManagerImpl(ReconDBProvider reconDBProvider) + throws IOException { namespaceDbStore = reconDBProvider.getDbStore(); - initializeTable(); + this.nsSummaryTable = NAMESPACE_SUMMARY.getTable(namespaceDbStore); } @Override public void initNSSummaryTable() throws IOException { - initializeTable(); + truncateTable(nsSummaryTable); } @Override @@ -64,17 +61,6 @@ public NSSummary getNSSummary(long objectId) throws IOException { return nsSummaryTable.get(objectId); } - private void initializeTable() { - try { - truncateTable(this.nsSummaryTable); - if (nsSummaryTable == null) { - this.nsSummaryTable = NAMESPACE_SUMMARY.getTable(namespaceDbStore); - } - } catch (IOException e) { - LOG.error("cannot initialize table"); - } - } - public Table getNSSummaryTable() { return nsSummaryTable; } From fb091524815ab07835ded4d3f8a5d58da44ef27a Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Fri, 25 Jun 2021 15:08:50 -0400 Subject: [PATCH 09/15] Update hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java Co-authored-by: Siyao Meng <50227127+smengcl@users.noreply.github.com> --- .../main/java/org/apache/hadoop/ozone/recon/ReconConstants.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index d02a2c1bf90f..461619ed58d1 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -58,6 +58,7 @@ private ReconConstants() { + PIPELINE_DB_SUFFIX; public static final String RECON_SCM_NODE_DB = "recon-node.db"; + // 1125899906842624L = 1PB public static final long MAX_FILE_SIZE_UPPER_BOUND = 1125899906842624L; public static final long MIN_FILE_SIZE_UPPER_BOUND = 1024L; public static final int NUM_OF_BINS = (int)(Math.log( From bb655353abd4032c6f9fd7864386c219ab004287 Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Fri, 25 Jun 2021 15:26:01 -0400 Subject: [PATCH 10/15] Update hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java Co-authored-by: Siyao Meng <50227127+smengcl@users.noreply.github.com> --- .../java/org/apache/hadoop/ozone/recon/ReconConstants.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java index 461619ed58d1..2eeb3235af32 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconConstants.java @@ -61,7 +61,8 @@ private ReconConstants() { // 1125899906842624L = 1PB public static final long MAX_FILE_SIZE_UPPER_BOUND = 1125899906842624L; public static final long MIN_FILE_SIZE_UPPER_BOUND = 1024L; - public static final int NUM_OF_BINS = (int)(Math.log( - MAX_FILE_SIZE_UPPER_BOUND) / Math.log(2) - Math.log( - MIN_FILE_SIZE_UPPER_BOUND) / Math.log(2)) + 1; + // 41 bins + public static final int NUM_OF_BINS = (int) Math.ceil(Math.log( + (double) MAX_FILE_SIZE_UPPER_BOUND / MIN_FILE_SIZE_UPPER_BOUND) / + Math.log(2)) + 1; } From 7fd9bec6e0ddc8569fbd393c98729723159ddb90 Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Fri, 25 Jun 2021 15:26:10 -0400 Subject: [PATCH 11/15] Update hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java Co-authored-by: Siyao Meng <50227127+smengcl@users.noreply.github.com> --- .../org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index bb8bdcc11ca6..267fd65f5bb0 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -34,7 +34,7 @@ public class NSSummaryCodec implements Codec { private final Codec integerCodec = new IntegerCodec(); - // 2 int fields + 40-length int array + // 2 int fields + 41-length int array private static final int NUM_OF_INTS = 2 + ReconConstants.NUM_OF_BINS; @Override From 0cb9d60567cdc976ece3ac1c62851fe4f254160e Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Fri, 25 Jun 2021 17:05:38 -0400 Subject: [PATCH 12/15] codec length change --- .../hadoop/hdds/utils/db/ShortCodec.java | 43 +++++++++++++++++++ .../recon/TestReconWithOzoneManagerHA.java | 2 +- .../hadoop/ozone/recon/ReconServer.java | 2 +- .../ozone/recon/codec/NSSummaryCodec.java | 10 +++-- .../ozone/recon/tasks/FileSizeCountTask.java | 1 - 5 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/ShortCodec.java diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/ShortCodec.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/ShortCodec.java new file mode 100644 index 000000000000..a7c4c068ef60 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/ShortCodec.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.utils.db; + +import java.io.IOException; +import com.google.common.primitives.Shorts; + +/** + * Codec to convert Short to/from byte array. + */ +public class ShortCodec implements Codec { + + @Override + public byte[] toPersistedFormat(Short object) throws IOException { + return Shorts.toByteArray(object); + } + + @Override + public Short fromPersistedFormat(byte[] rawData) throws IOException { + return Shorts.fromByteArray(rawData); + } + + @Override + public Short copyObject(Short object) { + return object; + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java index 3903275384ce..14ec9ff0cd19 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java @@ -131,7 +131,7 @@ public void testReconGetsSnapshotFromLeader() throws Exception { impl.syncDataFromOM(); ReconContainerMetadataManager reconContainerMetadataManager = - cluster.getReconServer().getRCMM(); + cluster.getReconServer().getReconContainerMetadataManager(); TableIterator iterator = reconContainerMetadataManager.getContainerTableIterator(); String reconKeyPrefix = null; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index d2cde334b451..c9030c0d3906 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -241,7 +241,7 @@ public StorageContainerServiceProvider getStorageContainerServiceProvider() { } @VisibleForTesting - public ReconContainerMetadataManager getRCMM() { + public ReconContainerMetadataManager getReconContainerMetadataManager() { return reconContainerMetadataManager; } diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index 267fd65f5bb0..97afac34e8c3 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.recon.codec; import org.apache.hadoop.hdds.utils.db.IntegerCodec; +import org.apache.hadoop.hdds.utils.db.ShortCodec; import org.apache.hadoop.ozone.recon.ReconConstants; import org.apache.hadoop.ozone.recon.api.types.NSSummary; import org.apache.hadoop.hdds.utils.db.Codec; @@ -34,15 +35,17 @@ public class NSSummaryCodec implements Codec { private final Codec integerCodec = new IntegerCodec(); + private final Codec shortCodec = new ShortCodec(); // 2 int fields + 41-length int array private static final int NUM_OF_INTS = 2 + ReconConstants.NUM_OF_BINS; @Override public byte[] toPersistedFormat(NSSummary object) throws IOException { - final int sizeOfRes = NUM_OF_INTS * Integer.BYTES; + final int sizeOfRes = NUM_OF_INTS * Integer.BYTES + Short.BYTES; ByteArrayOutputStream out = new ByteArrayOutputStream(sizeOfRes); out.write(integerCodec.toPersistedFormat(object.getNumOfFiles())); out.write(integerCodec.toPersistedFormat(object.getSizeOfFiles())); + out.write(shortCodec.toPersistedFormat((short)ReconConstants.NUM_OF_BINS)); int[] fileSizeBucket = object.getFileSizeBucket(); for (int i = 0; i < ReconConstants.NUM_OF_BINS; ++i) { out.write(integerCodec.toPersistedFormat(fileSizeBucket[i])); @@ -57,8 +60,9 @@ public NSSummary fromPersistedFormat(byte[] rawData) throws IOException { NSSummary res = new NSSummary(); res.setNumOfFiles(in.readInt()); res.setSizeOfFiles(in.readInt()); - int[] fileSizeBucket = new int[ReconConstants.NUM_OF_BINS]; - for (int i = 0; i < ReconConstants.NUM_OF_BINS && in.available() > 0; ++i) { + short len = in.readShort(); + int[] fileSizeBucket = new int[len]; + for (int i = 0; i < len; ++i) { fileSizeBucket[i] = in.readInt(); } res.setFileSizeBucket(fileSizeBucket); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java index f4cd6561b7f0..b561d4329b57 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java @@ -53,7 +53,6 @@ public class FileSizeCountTask implements ReconOmTask { private static final Logger LOG = LoggerFactory.getLogger(FileSizeCountTask.class); - // 1125899906842624L = 1PB private FileCountBySizeDao fileCountBySizeDao; private DSLContext dslContext; From 28985e543f7f79d987fbbe33fb069342cefda50b Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Fri, 25 Jun 2021 17:16:12 -0400 Subject: [PATCH 13/15] codec length change, minor fix --- .../org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index 97afac34e8c3..0e1d7db30594 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -55,7 +55,7 @@ public byte[] toPersistedFormat(NSSummary object) throws IOException { @Override public NSSummary fromPersistedFormat(byte[] rawData) throws IOException { - assert(rawData.length == NUM_OF_INTS * Integer.BYTES); + assert(rawData.length == NUM_OF_INTS * Integer.BYTES + Short.BYTES); DataInputStream in = new DataInputStream(new ByteArrayInputStream(rawData)); NSSummary res = new NSSummary(); res.setNumOfFiles(in.readInt()); From ec1fadba9c812eac4d2069423583bbff1222d87b Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Fri, 25 Jun 2021 18:23:56 -0400 Subject: [PATCH 14/15] Update hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java Co-authored-by: Siyao Meng <50227127+smengcl@users.noreply.github.com> --- .../java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index 0e1d7db30594..ba0fa333591a 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -61,6 +61,7 @@ public NSSummary fromPersistedFormat(byte[] rawData) throws IOException { res.setNumOfFiles(in.readInt()); res.setSizeOfFiles(in.readInt()); short len = in.readShort(); + assert(len == (short) NUM_OF_BINS); int[] fileSizeBucket = new int[len]; for (int i = 0; i < len; ++i) { fileSizeBucket[i] = in.readInt(); From 49223153635748239343f24c9cbbce63aa861791 Mon Sep 17 00:00:00 2001 From: Yuan Gu <53324985+yuangu002@users.noreply.github.com> Date: Fri, 25 Jun 2021 18:47:13 -0400 Subject: [PATCH 15/15] assert fix --- .../org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java index ba0fa333591a..389434556b6f 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/codec/NSSummaryCodec.java @@ -61,7 +61,7 @@ public NSSummary fromPersistedFormat(byte[] rawData) throws IOException { res.setNumOfFiles(in.readInt()); res.setSizeOfFiles(in.readInt()); short len = in.readShort(); - assert(len == (short) NUM_OF_BINS); + assert(len == (short) ReconConstants.NUM_OF_BINS); int[] fileSizeBucket = new int[len]; for (int i = 0; i < len; ++i) { fileSizeBucket[i] = in.readInt();