diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 211c6aa07c93..c3c84b66b284 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -62,8 +62,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; @@ -322,12 +320,7 @@ public static Collection getSCMAddresses( } return addresses; } - - public static boolean isHddsEnabled(Configuration conf) { - return conf.getBoolean(OZONE_ENABLED, OZONE_ENABLED_DEFAULT); - } - - + /** * Returns the hostname for this datanode. If the hostname is not * explicitly configured in the given config, then it is determined diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 3f7d0b915d5d..2efe1e2a1c67 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -74,9 +74,6 @@ public final class OzoneConfigKeys { "dfs.container.ratis.ipc.random.port"; public static final boolean DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT = false; - public static final String OZONE_ENABLED = - "ozone.enabled"; - public static final boolean OZONE_ENABLED_DEFAULT = false; public static final String OZONE_TRACE_ENABLED_KEY = "ozone.trace.enabled"; public static final boolean OZONE_TRACE_ENABLED_DEFAULT = false; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index b0a59fa209cc..d9383810250b 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -476,20 +476,6 @@ Socket timeout for Ozone client. Unit could be defined with postfix (ns,ms,s,m,h,d) - - ozone.enabled - false - OZONE, REQUIRED - - Status of the Ozone Object Storage service is enabled. - Set to true to enable Ozone. - Set to false to disable Ozone. - Unless this value is set to true, Ozone services will not be started in - the cluster. - - Please note: By default ozone is disabled on a hadoop cluster. - - ozone.key.deleting.limit.per.task 1000 diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index b13c37dd4534..970d6fd6ea23 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -171,62 +171,60 @@ public void start(OzoneConfiguration configuration) { public void start() { OzoneConfiguration.activate(); HddsUtils.initializeMetrics(conf, "HddsDatanode"); - if (HddsUtils.isHddsEnabled(conf)) { - try { - String hostname = HddsUtils.getHostName(conf); - String ip = InetAddress.getByName(hostname).getHostAddress(); - datanodeDetails = initializeDatanodeDetails(); - datanodeDetails.setHostName(hostname); - datanodeDetails.setIpAddress(ip); - TracingUtil.initTracing( - "HddsDatanodeService." + datanodeDetails.getUuidString() - .substring(0, 8)); - LOG.info("HddsDatanodeService host:{} ip:{}", hostname, ip); - // Authenticate Hdds Datanode service if security is enabled - if (OzoneSecurityUtil.isSecurityEnabled(conf)) { - component = "dn-" + datanodeDetails.getUuidString(); - - dnCertClient = new DNCertificateClient(new SecurityConfig(conf), - datanodeDetails.getCertSerialId()); - - if (SecurityUtil.getAuthenticationMethod(conf).equals( - UserGroupInformation.AuthenticationMethod.KERBEROS)) { - LOG.info("Ozone security is enabled. Attempting login for Hdds " + - "Datanode user. Principal: {},keytab: {}", conf.get( - DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY), - conf.get(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY)); - - UserGroupInformation.setConfiguration(conf); - - SecurityUtil.login(conf, DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, - DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hostname); - } else { - throw new AuthenticationException(SecurityUtil. - getAuthenticationMethod(conf) + " authentication method not " + - "supported. Datanode user" + " login " + "failed."); - } - LOG.info("Hdds Datanode login successful."); - } - if (OzoneSecurityUtil.isSecurityEnabled(conf)) { - initializeCertificateClient(conf); - } - datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf, - dnCertClient, this::terminateDatanode); - try { - httpServer = new HddsDatanodeHttpServer(conf); - httpServer.start(); - } catch (Exception ex) { - LOG.error("HttpServer failed to start.", ex); + try { + String hostname = HddsUtils.getHostName(conf); + String ip = InetAddress.getByName(hostname).getHostAddress(); + datanodeDetails = initializeDatanodeDetails(); + datanodeDetails.setHostName(hostname); + datanodeDetails.setIpAddress(ip); + TracingUtil.initTracing( + "HddsDatanodeService." + datanodeDetails.getUuidString() + .substring(0, 8)); + LOG.info("HddsDatanodeService host:{} ip:{}", hostname, ip); + // Authenticate Hdds Datanode service if security is enabled + if (OzoneSecurityUtil.isSecurityEnabled(conf)) { + component = "dn-" + datanodeDetails.getUuidString(); + + dnCertClient = new DNCertificateClient(new SecurityConfig(conf), + datanodeDetails.getCertSerialId()); + + if (SecurityUtil.getAuthenticationMethod(conf).equals( + UserGroupInformation.AuthenticationMethod.KERBEROS)) { + LOG.info("Ozone security is enabled. Attempting login for Hdds " + + "Datanode user. Principal: {},keytab: {}", conf.get( + DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY), + conf.get(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY)); + + UserGroupInformation.setConfiguration(conf); + + SecurityUtil.login(conf, DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, + DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hostname); + } else { + throw new AuthenticationException(SecurityUtil. + getAuthenticationMethod(conf) + " authentication method not " + + "supported. Datanode user" + " login " + "failed."); } - startPlugins(); - // Starting HDDS Daemons - datanodeStateMachine.startDaemon(); - } catch (IOException e) { - throw new RuntimeException("Can't start the HDDS datanode plugin", e); - } catch (AuthenticationException ex) { - throw new RuntimeException("Fail to authentication when starting" + - " HDDS datanode plugin", ex); + LOG.info("Hdds Datanode login successful."); + } + if (OzoneSecurityUtil.isSecurityEnabled(conf)) { + initializeCertificateClient(conf); + } + datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf, + dnCertClient, this::terminateDatanode); + try { + httpServer = new HddsDatanodeHttpServer(conf); + httpServer.start(); + } catch (Exception ex) { + LOG.error("HttpServer failed to start.", ex); } + startPlugins(); + // Starting HDDS Daemons + datanodeStateMachine.startDaemon(); + } catch (IOException e) { + throw new RuntimeException("Can't start the HDDS datanode plugin", e); + } catch (AuthenticationException ex) { + throw new RuntimeException("Fail to authentication when starting" + + " HDDS datanode plugin", ex); } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java index af56d0643d5b..c60ac66151ff 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.ozone; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -47,7 +46,6 @@ public class TestHddsDatanodeService { public void setUp() { testDir = GenericTestUtils.getRandomizedTestDir(); conf = new OzoneConfiguration(); - conf.setBoolean(OZONE_ENABLED, true); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath()); conf.setClass(OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY, MockService.class, ServicePlugin.class); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java index 04fd3a499aa0..eeca6b28dde7 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java @@ -72,7 +72,6 @@ public class TestHddsSecureDatanodeInit { public static void setUp() throws Exception { testDir = GenericTestUtils.getRandomizedTestDir(); conf = new OzoneConfiguration(); - conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath()); //conf.set(ScmConfigKeys.OZONE_SCM_NAMES, "localhost"); String volumeDir = testDir + "/disk1"; diff --git a/hadoop-hdds/docs/content/beyond/Containers.md b/hadoop-hdds/docs/content/beyond/Containers.md index 154d94dd2477..b2e3e0fd054d 100644 --- a/hadoop-hdds/docs/content/beyond/Containers.md +++ b/hadoop-hdds/docs/content/beyond/Containers.md @@ -113,7 +113,6 @@ The docker-config file contains the list of the required environment variables: OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True #... ``` diff --git a/hadoop-hdds/docs/content/start/OnPrem.md b/hadoop-hdds/docs/content/start/OnPrem.md index 3bf40a6a767f..243743acdb74 100644 --- a/hadoop-hdds/docs/content/start/OnPrem.md +++ b/hadoop-hdds/docs/content/start/OnPrem.md @@ -51,21 +51,6 @@ Let us look at the settings inside the generated file (ozone-site.xml) and how they control ozone. Once the right values are defined, this file needs to be copied to ```ozone directory/etc/hadoop```. - -* **ozone.enabled** This is the most critical setting for ozone. -Ozone is a work in progress and users have to enable this service explicitly. -By default, Ozone is disabled. Setting this flag to `true` enables ozone in the -HDFS or Ozone cluster. - -Here is an example, - -{{< highlight xml >}} - - ozone.enabled - true - -{{< /highlight >}} - * **ozone.metadata.dirs** Allows Administrators to specify where the metadata must reside. Usually you pick your fastest disk (SSD if you have them on your nodes). OzoneManager, SCM and datanode will write the @@ -124,7 +109,6 @@ Here is an example, | Setting | Value | Comment | |--------------------------------|------------------------------|------------------------------------------------------------------| -| ozone.enabled | true | This enables SCM and containers in HDFS cluster. | | ozone.metadata.dirs | file path | The metadata will be stored here. | | ozone.scm.names | SCM server name | Hostname:port or IP:port address of SCM. | | ozone.scm.block.client.address | SCM server name and port | Used by services like OM | diff --git a/hadoop-hdds/framework/src/main/resources/webapps/datanode/dn.js b/hadoop-hdds/framework/src/main/resources/webapps/datanode/dn.js index 3b671671de22..f9c7e7c1b1fd 100644 --- a/hadoop-hdds/framework/src/main/resources/webapps/datanode/dn.js +++ b/hadoop-hdds/framework/src/main/resources/webapps/datanode/dn.js @@ -34,7 +34,6 @@ $.get('/jmx?qry=Hadoop:service=OzoneDataNode,name=SCMConnectionManager', function (resp) { if (resp.beans.length > 0) { data.ozone.SCMServers = resp.beans[0].SCMServers; - data.ozone.enabled = true; render(); } }).fail(show_err_msg); @@ -44,7 +43,6 @@ $.get('/jmx?qry=Hadoop:service=OzoneDataNode,name=ContainerLocationManager', function (resp) { if (resp.beans.length > 0) { data.ozone.LocationReport = resp.beans[0].LocationReport; - data.ozone.enabled = true; render(); } }).fail(show_err_msg); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index af65e1365bf7..7a375fcd039e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -118,7 +118,6 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; /** * StorageContainerManager is the main entry point for the service that @@ -594,15 +593,8 @@ public static RPC.Server startRpcServer( * @return SCM instance * @throws IOException, AuthenticationException */ - public static StorageContainerManager createSCM( - OzoneConfiguration conf) + public static StorageContainerManager createSCM(OzoneConfiguration conf) throws IOException, AuthenticationException { - if (!HddsUtils.isHddsEnabled(conf)) { - System.err.println( - "SCM cannot be started in secure mode or when " + OZONE_ENABLED + "" + - " is set to false"); - System.exit(1); - } return new StorageContainerManager(conf); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java index 38f78ad81535..8f6dc3b49d38 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/HddsTestUtils.java @@ -34,8 +34,6 @@ import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; - /** * Stateless helper functions for Hdds tests. */ @@ -81,7 +79,6 @@ public static StorageContainerManager getScm(OzoneConfiguration conf) conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, "127.0.0.1:0"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0"); conf.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "127.0.0.1:0"); - conf.setBoolean(OZONE_ENABLED, true); SCMStorageConfig scmStore = new SCMStorageConfig(conf); if(scmStore.getState() != Storage.StorageState.INITIALIZED) { String clusterId = UUID.randomUUID().toString(); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java index 37321d76996b..ea5d571d8501 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java @@ -75,8 +75,6 @@ import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; - /** * Stateless helper functions to handler scm/datanode connection. */ @@ -530,7 +528,6 @@ public static StorageContainerManager getScm(OzoneConfiguration conf) public static StorageContainerManager getScm(OzoneConfiguration conf, SCMConfigurator configurator) throws IOException, AuthenticationException { - conf.setBoolean(OZONE_ENABLED, true); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0"); conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, "127.0.0.1:0"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0"); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java index 5982b4fb4c83..c9166ab5a4dc 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java @@ -70,7 +70,6 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys .OZONE_SCM_BLOCK_DELETION_MAX_RETRY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.when; @@ -91,7 +90,6 @@ public void setup() throws Exception { testDir = GenericTestUtils.getTestDir( TestDeletedBlockLog.class.getSimpleName()); conf = new OzoneConfiguration(); - conf.set(OZONE_ENABLED, "true"); conf.setInt(OZONE_SCM_BLOCK_DELETION_MAX_RETRY, 20); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath()); scm = TestUtils.getScm(conf); diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh index 9ddaab64a7fd..556dd856a091 100755 --- a/hadoop-ozone/common/src/main/bin/start-ozone.sh +++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh @@ -88,15 +88,6 @@ SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKe # exit 1 #fi -#--------------------------------------------------------- -# Check if ozone is enabled -OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-) -if [[ "${OZONE_ENABLED}" != "true" ]]; then - echo "Operation is not supported because ozone is not enabled." - exit -1 -fi - -#--------------------------------------------------------- # datanodes (using default workers file) echo "Starting datanodes" diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh index c07d42b42813..5693f81ef69e 100755 --- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh +++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh @@ -54,15 +54,6 @@ fi # exit 1 #fi -#--------------------------------------------------------- -# Check if ozone is enabled -OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-) -if [[ "${OZONE_ENABLED}" != "true" ]]; then - echo "Operation is not supported because ozone is not enabled." - exit -1 -fi - -#--------------------------------------------------------- # datanodes (using default workers file) echo "Stopping datanodes" diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java index e146d31afd5c..0a25065f1035 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java @@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.ozone.OzoneConsts; @@ -132,12 +131,7 @@ public static long formatDate(String date) throws ParseException { Preconditions.checkNotNull(date, "Date string should not be null."); return DATE_FORMAT.get().parse(date).getTime(); } - - public static boolean isOzoneEnabled(Configuration conf) { - return HddsUtils.isHddsEnabled(conf); - } - - + /** * verifies that bucket name / volume name is a valid DNS name. * diff --git a/hadoop-ozone/dev-support/intellij/ozone-site.xml b/hadoop-ozone/dev-support/intellij/ozone-site.xml index 2eb79aa5219e..9083a38cccb9 100644 --- a/hadoop-ozone/dev-support/intellij/ozone-site.xml +++ b/hadoop-ozone/dev-support/intellij/ozone-site.xml @@ -23,10 +23,6 @@ ozone.scm.block.client.address localhost - - ozone.enabled - True - ozone.scm.datanode.id /tmp/datanode.id @@ -47,10 +43,6 @@ ozone.om.address localhost - - ozone.enabled - true - ozone.scm.container.size 10MB diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config index 63bbbd898733..0a73daf1cb30 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config @@ -17,7 +17,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=true OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config index f7ea7d5245fd..ee7a773cc6d5 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config @@ -17,7 +17,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=true OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config index 39bfa17f7dd4..4122daeef5d3 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config @@ -23,7 +23,6 @@ OZONE-SITE.XML_ozone.om.address.id1.om2=om2 OZONE-SITE.XML_ozone.om.address.id1.om3=om3 OZONE-SITE.XML_ozone.om.ratis.enable=true OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config index f3de99a50a79..a727e3fe24ce 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config @@ -23,7 +23,6 @@ OZONE-SITE.XML_ozone.om.address.id1.om2=om2 OZONE-SITE.XML_ozone.om.address.id1.om3=om3 OZONE-SITE.XML_ozone.om.ratis.enable=true OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config index c9848d54843a..bf7871e0b5a9 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config @@ -17,7 +17,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config index 5445c9c116b7..ec9c59059d4b 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config @@ -17,7 +17,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index a8c2a4d98675..d82cde2ea386 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -17,7 +17,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config index 4d5466c6ab9e..3625fc6b09ac 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config @@ -17,7 +17,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config index e3614f97d86d..931f112508d2 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config @@ -17,7 +17,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=true OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config index 306e2a89770a..75d197759b2e 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config @@ -16,7 +16,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=true OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config index 306e2a89770a..75d197759b2e 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config @@ -16,7 +16,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=true OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config index fe713e0dde21..995089813790 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config @@ -17,7 +17,6 @@ CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 OZONE-SITE.XML_ozone.ksm.address=ksm OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=true OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index c6ba94777807..7d6fdc4dd0fa 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -17,7 +17,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index 86634af23832..ff4224910922 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -19,7 +19,6 @@ CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index 5f5e70b5cdf8..6e638915a247 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -26,7 +26,6 @@ data: OZONE-SITE.XML_ozone.om.address: "om-0.om" OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm" OZONE-SITE.XML_ozone.scm.names: "scm-0.scm" - OZONE-SITE.XML_ozone.enabled: "true" LOG4J.PROPERTIES_log4j.rootLogger: "INFO, stdout" LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender" LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml index 94d16d2f74d6..00fb72b14e50 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml @@ -26,7 +26,6 @@ data: OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_ozone.enabled: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml index 94d16d2f74d6..00fb72b14e50 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml @@ -26,7 +26,6 @@ data: OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_ozone.enabled: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml index 43d11a48c9ad..82f303fafe4e 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -26,7 +26,6 @@ data: OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_ozone.enabled: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml index e554145bac69..c7e8f486e89f 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml @@ -26,7 +26,6 @@ data: OZONE-SITE.XML_ozone.om.address: om-0.om OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm OZONE-SITE.XML_ozone.scm.names: scm-0.scm - OZONE-SITE.XML_ozone.enabled: "true" LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout diff --git a/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config index 1db1a798d359..da46f8e0b3ab 100644 --- a/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config +++ b/hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config @@ -17,7 +17,6 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index 0aba9689ffbe..59cef37d5364 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -252,8 +252,7 @@ abstract class Builder { protected Optional hbProcessorInterval = Optional.empty(); protected Optional scmId = Optional.empty(); protected Optional omId = Optional.empty(); - - protected Boolean ozoneEnabled = true; + protected Boolean randomContainerPort = true; protected Optional chunkSize = Optional.empty(); protected Optional streamBufferFlushSize = Optional.empty(); @@ -390,16 +389,6 @@ public Builder setTrace(Boolean trace) { return this; } - /** - * Modifies the configuration such that Ozone will be disabled. - * - * @return MiniOzoneCluster.Builder - */ - public Builder disableOzone() { - ozoneEnabled = false; - return this; - } - /** * Sets the chunk size. * diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index ac76482bd908..8b2fc9200a68 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -465,7 +465,6 @@ public MiniOzoneCluster build() throws IOException { * @throws IOException */ void initializeConfiguration() throws IOException { - conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, ozoneEnabled); Path metaDir = Paths.get(path, "ozone-meta"); Files.createDirectories(metaDir); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString()); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index ca1f17934582..b38a7cbb5b1f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -95,7 +95,6 @@ import static junit.framework.TestCase.assertNotNull; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER; @@ -235,7 +234,6 @@ private void stopMiniKdc() { private void setSecureConfig(Configuration configuration) throws IOException { configuration.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - configuration.setBoolean(OZONE_ENABLED, true); host = InetAddress.getLocalHost().getCanonicalHostName() .toLowerCase(); String realm = miniKdc.getRealm(); @@ -339,7 +337,6 @@ private void initSCM() temp.mkdirs(); } conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString()); - conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true); SCMStorageConfig scmStore = new SCMStorageConfig(conf); scmStore.setClusterId(clusterId); scmStore.setScmId(scmId); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java index ba072f81e9fd..41e22c737b21 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java @@ -478,7 +478,6 @@ public void testSCMInitializationFailure() GenericTestUtils.getTempPath(UUID.randomUUID().toString()); Path scmPath = Paths.get(path, "scm-meta"); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString()); - conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true); exception.expect(SCMException.class); exception.expectMessage( "SCM not initialized due to storage config failure"); @@ -493,7 +492,6 @@ public void testScmInfo() throws Exception { try { Path scmPath = Paths.get(path, "scm-meta"); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString()); - conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true); SCMStorageConfig scmStore = new SCMStorageConfig(conf); String clusterId = UUID.randomUUID().toString(); String scmId = UUID.randomUUID().toString(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java index 2716d51f07e9..249112435aff 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerConfiguration.java @@ -23,7 +23,6 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OmUtils; -import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneIllegalArgumentException; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.test.GenericTestUtils; @@ -65,7 +64,6 @@ public void init() throws IOException { scmId = UUID.randomUUID().toString(); final String path = GenericTestUtils.getTempPath(omId); Path metaDirPath = Paths.get(path, "om-meta"); - conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString()); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0"); conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java index 65bc2751423e..740baec4fa72 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java @@ -17,32 +17,20 @@ */ package org.apache.hadoop.ozone.ozShell; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; +import static org.junit.Assert.fail; + import java.util.Arrays; import java.util.List; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hdds.client.ReplicationFactor; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.HddsDatanodeService; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.test.GenericTestUtils; - -import com.google.common.base.Strings; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; -import org.junit.After; -import org.junit.AfterClass; import org.junit.Assert; -import static org.junit.Assert.fail; -import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import com.google.common.base.Strings; + import picocli.CommandLine; import picocli.CommandLine.ExecutionException; import picocli.CommandLine.IExceptionHandler2; @@ -58,79 +46,18 @@ public class TestOzoneDatanodeShell { private static final Logger LOG = LoggerFactory.getLogger(TestOzoneDatanodeShell.class); - /** - * Set the timeout for every test. - */ - @Rule - public Timeout testTimeout = new Timeout(300000); - - private static File baseDir; - private static OzoneConfiguration conf = null; - private static MiniOzoneCluster cluster = null; private static HddsDatanodeService datanode = null; - private final ByteArrayOutputStream out = new ByteArrayOutputStream(); - private final ByteArrayOutputStream err = new ByteArrayOutputStream(); - private static final PrintStream OLD_OUT = System.out; - private static final PrintStream OLD_ERR = System.err; - /** - * Create a MiniDFSCluster for testing with using distributed Ozone - * handler type. + * Create a HddsDatanodeService stub that can be used to test Cli behavior. * * @throws Exception */ @BeforeClass - public static void init() throws Exception { - conf = new OzoneConfiguration(); - - String path = GenericTestUtils.getTempPath( - TestOzoneDatanodeShell.class.getSimpleName()); - baseDir = new File(path); - baseDir.mkdirs(); - - datanode = HddsDatanodeService.createHddsDatanodeService(null); - - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) - .build(); - conf.setInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue()); - conf.setQuietMode(false); - cluster.waitForClusterToBeReady(); + public static void init() { + datanode = new TestHddsDatanodeService(false, new String[] {}); } - - /** - * shutdown MiniDFSCluster. - */ - @AfterClass - public static void shutdown() { - if (cluster != null) { - cluster.shutdown(); - } - - if (baseDir != null) { - FileUtil.fullyDelete(baseDir, true); - } - } - - @Before - public void setup() { - System.setOut(new PrintStream(out)); - System.setErr(new PrintStream(err)); - } - - @After - public void reset() { - // reset stream after each unit test - out.reset(); - err.reset(); - - // restore system streams - System.setOut(OLD_OUT); - System.setErr(OLD_ERR); - } - - + private void executeDatanode(HddsDatanodeService hdds, String[] args) { LOG.info("Executing datanode command with args {}", Arrays.asList(args)); CommandLine cmd = hdds.getCmd(); @@ -201,4 +128,15 @@ public void testDatanodeInvalidParamCommand() { executeDatanodeWithError(datanode, args, expectedError); } -} \ No newline at end of file + + private static class TestHddsDatanodeService extends HddsDatanodeService { + TestHddsDatanodeService(boolean printBanner, String[] args) { + super(printBanner, args); + } + + @Override + public void start() { + // do nothing + } + } +} diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java index ab351913d7a0..fcabc67e785e 100644 --- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java +++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java @@ -28,7 +28,6 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.OmUtils; -import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -92,7 +91,6 @@ public void init() throws Exception { scmId = UUID.randomUUID().toString(); final String path = GenericTestUtils.getTempPath(omId); java.nio.file.Path metaDirPath = java.nio.file.Paths.get(path, "om-meta"); - conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString()); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0"); conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, true); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java index ae3a83b8f288..94557cc1668b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java @@ -109,9 +109,7 @@ public static void generateConfigurations(String path) throws for (OzoneConfiguration.Property p : allProperties) { if (p.getTag() != null && p.getTag().contains("REQUIRED")) { - if (p.getName().equalsIgnoreCase(OzoneConfigKeys.OZONE_ENABLED)) { - p.setValue(String.valueOf(Boolean.TRUE)); - } else if (p.getName().equalsIgnoreCase( + if (p.getName().equalsIgnoreCase( OzoneConfigKeys.OZONE_METADATA_DIRS)) { p.setValue(System.getProperty(OzoneConsts.JAVA_TMP_DIR)); } else if (p.getName().equalsIgnoreCase( diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOMClient.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOMClient.java index cfc1e1ebdaaf..dfbecf99375f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOMClient.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOMClient.java @@ -18,6 +18,14 @@ package org.apache.hadoop.ozone.genesis; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.locks.ReentrantLock; + import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -30,22 +38,23 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.OMConfigKeys; -import org.apache.hadoop.ozone.om.helpers.*; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ratis.protocol.ClientId; -import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.infra.Blackhole; -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.*; -import java.util.concurrent.locks.ReentrantLock; - -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; - /** * Benchmarks OM Client. */ @@ -68,7 +77,6 @@ public static void initialize() throws IOException { if (!bool) { bool = true; OzoneConfiguration conf = new OzoneConfiguration(); - conf.setBoolean(OZONE_ENABLED, true); testDir = GenesisUtil.getTempPath() .resolve(RandomStringUtils.randomNumeric(7)).toString(); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java index 45c90d3bc7d9..2e2b6105da78 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkOzoneManager.java @@ -18,6 +18,15 @@ package org.apache.hadoop.ozone.genesis; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.locks.ReentrantLock; + import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -25,28 +34,29 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; -import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.ozone.om.OzoneManager; -import org.apache.hadoop.ozone.om.helpers.*; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.infra.Blackhole; -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.locks.ReentrantLock; - -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; - /** * Benchmarks OzoneManager. */ @@ -72,7 +82,6 @@ public static void initialize() lock.lock(); if (scm == null) { OzoneConfiguration conf = new OzoneConfiguration(); - conf.setBoolean(OZONE_ENABLED, true); testDir = GenesisUtil.getTempPath() .resolve(RandomStringUtils.randomNumeric(7)).toString(); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java index 6ec0833cc3f5..f8c74dd67afe 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkSCM.java @@ -18,9 +18,12 @@ package org.apache.hadoop.ozone.genesis; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; + import java.io.File; import java.io.IOException; import java.util.concurrent.locks.ReentrantLock; + import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -28,20 +31,24 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.scm.block.BlockManager; -import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; +import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.infra.Blackhole; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; - /** * Benchmarks BlockManager class. */ @@ -65,7 +72,6 @@ public static void initialize() lock.lock(); if (scm == null) { OzoneConfiguration conf = new OzoneConfiguration(); - conf.setBoolean(OZONE_ENABLED, true); testDir = GenesisUtil.getTempPath() .resolve(RandomStringUtils.randomNumeric(7)).toString(); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir);