From 800e3c5d171bc95368b0d5ea793103c8c109303c Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Wed, 17 Mar 2021 12:31:29 +0530 Subject: [PATCH 01/18] HDDS-4915. [SCM HA Security] Integrate CertClient. --- .../hadoop/hdds/scm/XceiverClientGrpc.java | 10 +- .../hadoop/hdds/scm/XceiverClientManager.java | 17 +- .../hadoop/hdds/scm/XceiverClientRatis.java | 2 +- .../apache/hadoop/hdds/ratis/RatisHelper.java | 2 +- .../apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 17 + .../hadoop/hdds/scm/ha/SCMNodeInfo.java | 17 + .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../hadoop/ozone/HddsDatanodeService.java | 6 + .../server/ratis/XceiverServerRatis.java | 8 +- .../container/ozoneimpl/OzoneContainer.java | 25 +- ...ecurityProtocolClientSideTranslatorPB.java | 10 +- .../authority/CertificateStore.java | 3 + .../authority/DefaultApprover.java | 5 +- .../certificate/client/CertificateClient.java | 20 ++ .../client/DefaultCertificateClient.java | 108 +++++-- .../org/apache/hadoop/hdds/utils/HAUtils.java | 129 ++++++++ .../hadoop/hdds/utils/HddsServerUtil.java | 2 +- .../certificate/authority/MockCAStore.java | 4 + .../hadoop/hdds/scm/ha/HASecurityUtils.java | 29 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 5 +- .../hdds/scm/ha/SCMRatisServerImpl.java | 13 +- ...ecurityProtocolServerSideTranslatorPB.java | 14 +- .../hadoop/hdds/scm/server/SCMCertStore.java | 2 +- .../scm/server/SCMSecurityProtocolServer.java | 39 +-- .../hdds/scm/server/SCMStarterInterface.java | 4 +- .../hdds/scm/server/SCMStorageConfig.java | 19 ++ .../scm/server/StorageContainerManager.java | 290 ++++++++++++++---- .../StorageContainerManagerStarter.java | 3 +- .../scm/cli/ContainerOperationClient.java | 12 +- .../apache/hadoop/hdds/scm/cli/ScmOption.java | 1 + .../hadoop/ozone/client/rpc/RpcClient.java | 8 +- .../ozone/om/helpers/ServiceInfoEx.java | 8 +- ...ManagerProtocolClientSideTranslatorPB.java | 2 +- .../ozonesecure-om-ha/docker-compose.yaml | 94 +++++- .../compose/ozonesecure-om-ha/docker-config | 13 +- .../main/compose/ozonesecure-om-ha/test.sh | 7 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 13 +- .../client/CertificateClientTestImpl.java | 20 ++ .../ozoneimpl/TestOzoneContainerWithTLS.java | 3 +- .../src/main/proto/OmClientProtocol.proto | 2 + .../apache/hadoop/ozone/om/OzoneManager.java | 18 +- .../om/ratis/OzoneManagerRatisServer.java | 8 +- .../OzoneManagerRequestHandler.java | 5 + 43 files changed, 834 insertions(+), 184 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java index 87030112cb4e..c3ec5d1de50c 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java @@ -82,7 +82,7 @@ public class XceiverClientGrpc extends XceiverClientSpi { private final long timeout; private SecurityConfig secConfig; private final boolean topologyAwareRead; - private X509Certificate caCert; + private List caCerts; // Cache the DN which returned the GetBlock command so that the ReadChunk // command can be sent to the same DN. private Map getBlockDNcache; @@ -96,7 +96,7 @@ public class XceiverClientGrpc extends XceiverClientSpi { * @param caCert - SCM ca certificate. */ public XceiverClientGrpc(Pipeline pipeline, ConfigurationSource config, - X509Certificate caCert) { + List caCert) { super(); Preconditions.checkNotNull(pipeline); Preconditions.checkNotNull(config); @@ -114,7 +114,7 @@ public XceiverClientGrpc(Pipeline pipeline, ConfigurationSource config, this.topologyAwareRead = config.getBoolean( OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_DEFAULT); - this.caCert = caCert; + this.caCerts = caCert; this.getBlockDNcache = new ConcurrentHashMap<>(); } @@ -179,8 +179,8 @@ private synchronized void connectToDatanode(DatanodeDetails dn) .intercept(new GrpcClientInterceptor()); if (secConfig.isGrpcTlsEnabled()) { SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient(); - if (caCert != null) { - sslContextBuilder.trustManager(caCert); + if (caCerts != null) { + sslContextBuilder.trustManager(caCerts); } if (secConfig.useTestCert()) { channelBuilder.overrideAuthority("localhost"); diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java index 31728c6a804d..341ab6f4f102 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; @@ -68,7 +70,7 @@ public class XceiverClientManager implements Closeable, XceiverClientFactory { //TODO : change this to SCM configuration class private final ConfigurationSource conf; private final Cache clientCache; - private X509Certificate caCert; + private List caCerts; private static XceiverClientMetrics metrics; private boolean isSecurityEnabled; @@ -86,16 +88,19 @@ public XceiverClientManager(ConfigurationSource conf) throws IOException { public XceiverClientManager(ConfigurationSource conf, ScmClientConfig clientConf, - String caCertPem) throws IOException { + List caCertPems) throws IOException { Preconditions.checkNotNull(clientConf); Preconditions.checkNotNull(conf); long staleThresholdMs = clientConf.getStaleThreshold(MILLISECONDS); this.conf = conf; this.isSecurityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf); if (isSecurityEnabled) { - Preconditions.checkNotNull(caCertPem); + Preconditions.checkNotNull(caCertPems); try { - this.caCert = CertificateCodec.getX509Cert(caCertPem); + this.caCerts = new ArrayList<>(caCertPems.size()); + for (String cert : caCertPems) { + this.caCerts.add(CertificateCodec.getX509Cert(cert)); + } } catch (CertificateException ex) { throw new SCMSecurityException("Error: Fail to get SCM CA certificate", ex); @@ -232,10 +237,10 @@ public XceiverClientSpi call() throws Exception { switch (type) { case RATIS: client = XceiverClientRatis.newXceiverClientRatis(pipeline, conf, - caCert); + caCerts); break; case STAND_ALONE: - client = new XceiverClientGrpc(pipeline, conf, caCert); + client = new XceiverClientGrpc(pipeline, conf, caCerts); break; case CHAINED: default: diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java index b6cd21602d4f..cece6a6fac50 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java @@ -81,7 +81,7 @@ public static XceiverClientRatis newXceiverClientRatis( public static XceiverClientRatis newXceiverClientRatis( org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline, - ConfigurationSource ozoneConf, X509Certificate caCert) { + ConfigurationSource ozoneConf, List caCert) { final String rpcType = ozoneConf .get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY, ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java index 06acb4495526..4b648a102330 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java @@ -294,7 +294,7 @@ private static Map getDatanodeRatisPrefixProps( // For External gRPC client to server with gRPC TLS. // No mTLS for external client as SCM CA does not issued certificates for them public static GrpcTlsConfig createTlsClientConfig(SecurityConfig conf, - X509Certificate caCert) { + List caCert) { GrpcTlsConfig tlsConfig = null; if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { tlsConfig = new GrpcTlsConfig(null, null, diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java index b3fc40e8b9c0..601821d596f1 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java @@ -56,6 +56,17 @@ public static boolean isSCMHAEnabled(ConfigurationSource conf) { ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT); } + public static boolean isHAWithMultiSCMNodes(ConfigurationSource conf) { + if (isSCMHAEnabled(conf)) { + String scmServiceId = SCMHAUtils.getScmServiceId(conf); + Collection nodeIds = SCMHAUtils.getSCMNodeIds(conf, scmServiceId); + if (nodeIds.size() > 1) { + return true; + } + } + return false; + } + public static String getPrimordialSCM(ConfigurationSource conf) { return conf.get(ScmConfigKeys.OZONE_SCM_PRIMORDIAL_NODE_ID_KEY); } @@ -173,4 +184,10 @@ public static OzoneConfiguration removeSelfId( } return conf; } + + public static Collection getSCMNodeIds( + ConfigurationSource configuration) { + String scmServiceId = SCMHAUtils.getScmServiceId(configuration); + return SCMHAUtils.getSCMNodeIds(configuration, scmServiceId); + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java index f3191ea41327..5dacf9ed8ae2 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.ha; +import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.ozone.ha.ConfUtils; @@ -246,4 +247,20 @@ public String getScmSecurityAddress() { public String getScmDatanodeAddress() { return scmDatanodeAddress; } + + public static SCMNodeInfo getScmNodeInfo(ConfigurationSource conf, + String nodeId) { + List scmNodeInfoList = SCMNodeInfo.buildNodeInfo(conf); + Preconditions.checkState(scmNodeInfoList.size() >=1); + if (SCMHAUtils.getScmServiceId(conf) != null) { + for (SCMNodeInfo scmNodeInfo : scmNodeInfoList) { + if (scmNodeInfo.getNodeId().equals(nodeId)) { + return scmNodeInfo; + } + } + } else { + return scmNodeInfoList.get(0); + } + return null; + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index b12a022187e7..90ea07339c30 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -41,6 +41,7 @@ public final class OzoneConsts { public static final String SCM_ID = "scmUuid"; public static final String CLUSTER_ID_PREFIX = "CID-"; public static final String SCM_CERT_SERIAL_ID = "scmCertSerialId"; + public static final String PRIMARY_SCM_NODE_ID = "primaryScmNodeId"; public static final String OZONE_SIMPLE_ROOT_USER = "root"; public static final String OZONE_SIMPLE_HDFS_USER = "hdfs"; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 325736e48456..12f95c773e2f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -355,6 +355,12 @@ private void getSCMSignedCert(OzoneConfiguration config) { dnCertClient.storeCertificate(pemEncodedCert, true); dnCertClient.storeCertificate(response.getX509CACertificate(), true, true); + + // Store Root CA certificate. + if (response.hasX509RootCACertificate()) { + dnCertClient.storeRootCACertificate( + response.getX509RootCACertificate(), true); + } String dnCertSerialId = getX509Certificate(pemEncodedCert). getSerialNumber().toString(); datanodeDetails.setCertSerialId(dnCertSerialId); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 72859e1e4620..84f3cae8d767 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -465,9 +466,14 @@ private static Parameters createTlsParameters(SecurityConfig conf, Parameters parameters = new Parameters(); if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { + ArrayList< X509Certificate > listCA = new ArrayList<>(); + listCA.add(caClient.getCACertificate()); + if (caClient.getRootCACertificate() != null) { + listCA.add(caClient.getRootCACertificate()); + } GrpcTlsConfig serverConfig = new GrpcTlsConfig( caClient.getPrivateKey(), caClient.getCertificate(), - caClient.getCACertificate(), true); + listCA, true); GrpcConfigKeys.Server.setTlsConf(parameters, serverConfig); GrpcConfigKeys.Admin.setTlsConf(parameters, serverConfig); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 8b37c5b0b07f..91ed37364743 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.container.ozoneimpl; import java.io.IOException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.time.Duration; import java.util.ArrayList; import java.util.Iterator; @@ -40,6 +42,8 @@ import org.apache.hadoop.hdds.security.token.BlockTokenVerifier; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; @@ -167,8 +171,25 @@ public OzoneContainer( blockDeletingService = new BlockDeletingService(this, svcInterval.toMillis(), serviceTimeout, TimeUnit.MILLISECONDS, config); - tlsClientConfig = RatisHelper.createTlsClientConfig( - secConf, certClient != null ? certClient.getCACertificate() : null); + + List< X509Certificate > x509Certificates; + if (certClient != null) { + List pemEncodedCerts = HAUtils.buildCAList(certClient, conf); + x509Certificates = new ArrayList<>(pemEncodedCerts.size()); + for (String cert : pemEncodedCerts) { + try { + x509Certificates.add(CertificateCodec.getX509Certificate(cert)); + } catch (CertificateException ex) { + LOG.error("Error while fetching CA", ex); + throw new IOException(ex); + } + } + } else { + x509Certificates = null; + } + + tlsClientConfig = RatisHelper.createTlsClientConfig(secConf, + x509Certificates); isStarted = new AtomicBoolean(false); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java index 0ab5d7e71fe8..67d4dafcc600 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java @@ -265,12 +265,16 @@ public SCMGetCertResponseProto getDataNodeCertificateChain( */ @Override public String getCACertificate() throws IOException { + return getCACert().getX509Certificate(); + } + + + public SCMGetCertResponseProto getCACert() throws IOException { SCMGetCACertificateRequestProto protoIns = SCMGetCACertificateRequestProto .getDefaultInstance(); return submitRequest(Type.GetCACertificate, builder -> builder.setGetCACertificateRequest(protoIns)) - .getGetCertResponseProto().getX509Certificate(); - + .getGetCertResponseProto(); } /** @@ -301,7 +305,7 @@ public List listCertificate(HddsProtos.NodeType role, public String getRootCACertificate() throws IOException { SCMGetCACertificateRequestProto protoIns = SCMGetCACertificateRequestProto .getDefaultInstance(); - return submitRequest(Type.GetCACertificate, + return submitRequest(Type.GetRootCACertificate, builder -> builder.setGetCACertificateRequest(protoIns)) .getGetCertResponseProto().getX509RootCACertificate(); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java index 0a669b9e9f02..2f3657d5963c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java @@ -54,6 +54,9 @@ public interface CertificateStore { void storeValidCertificate(BigInteger serialID, X509Certificate certificate, NodeType role) throws IOException; + void storeValidScmCertificate(BigInteger serialID, + X509Certificate certificate) throws IOException; + /** * Check certificate serialID exists or not. If exists throws an exception. * @param serialID diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultApprover.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultApprover.java index 39f610c0e4b4..3ab0911c2a8c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultApprover.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultApprover.java @@ -67,6 +67,7 @@ public DefaultApprover(PKIProfile pkiProfile, SecurityConfig config) { /** * Sign function signs a Certificate. + * * @param config - Security Config. * @param caPrivate - CAs private Key. * @param caCertificate - CA Certificate. @@ -76,12 +77,12 @@ public DefaultApprover(PKIProfile pkiProfile, SecurityConfig config) { * @param scmId - SCM id. * @param clusterId - Cluster id. * @return Signed Certificate. - * @throws IOException - On Error + * @throws IOException - On Error * @throws OperatorCreationException - on Error. */ @SuppressWarnings("ParameterNumber") @Override - public X509CertificateHolder sign( + public X509CertificateHolder sign( SecurityConfig config, PrivateKey caPrivate, X509CertificateHolder caCertificate, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java index 0ec4d4254cf6..78e3406cc607 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; +import java.io.IOException; import java.io.InputStream; import java.security.PrivateKey; import java.security.PublicKey; @@ -218,4 +219,23 @@ enum InitResponse { */ String getComponentName(); + /** + * Return the latest Root CA certificate known to the client. + * @return latest Root CA certificate known to the client. + */ + X509Certificate getRootCACertificate(); + + /** + * Store RootCA certificate. + * @param pemEncodedCert + * @param force + * @throws CertificateException + */ + void storeRootCACertificate(String pemEncodedCert, boolean force) + throws CertificateException; + + List listCA() throws IOException; + + List updateCAList() throws IOException; + } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index 1b043564a35c..fa9488b6662c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -23,7 +23,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -45,22 +44,14 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; -import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB; import org.apache.hadoop.hdds.utils.HddsServerUtil; -import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; import org.apache.hadoop.hdds.security.x509.keys.KeyCodec; -import org.apache.hadoop.ipc.Client; -import org.apache.hadoop.ipc.ProtobufRpcEngine; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OzoneSecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; import com.google.common.base.Preconditions; import org.apache.commons.io.FilenameUtils; @@ -88,6 +79,8 @@ public abstract class DefaultCertificateClient implements CertificateClient { private static final String CERT_FILE_NAME_FORMAT = "%s.crt"; private static final String CA_CERT_PREFIX = "CA-"; private static final int CA_CERT_PREFIX_LEN = 3; + private static final String ROOT_CA_CERT_PREFIX = "ROOTCA-"; + private static final int ROOT_CA_PREFIX_LEN = 7; private final Logger logger; private final SecurityConfig securityConfig; private final KeyCodec keyCodec; @@ -97,7 +90,9 @@ public abstract class DefaultCertificateClient implements CertificateClient { private Map certificateMap; private String certSerialId; private String caCertId; + private String rootCaCertId; private String component; + private List pemEncodedCACerts = null; DefaultCertificateClient(SecurityConfig securityConfig, Logger log, String certSerialId, String component) { @@ -127,6 +122,7 @@ private void loadAllCertificates() { CertificateCodec certificateCodec = new CertificateCodec(securityConfig, component); long latestCaCertSerailId = -1L; + long latestRootCaCertSerialId = -1L; for (File file : certFiles) { if (file.isFile()) { try { @@ -149,6 +145,16 @@ private void loadAllCertificates() { latestCaCertSerailId = tmpCaCertSerailId; } } + + if (file.getName().startsWith(ROOT_CA_CERT_PREFIX)) { + String certFileName = FilenameUtils.getBaseName( + file.getName()); + long tmpRootCaCertSerailId = NumberUtils.toLong( + certFileName.substring(ROOT_CA_PREFIX_LEN)); + if (tmpRootCaCertSerailId > latestRootCaCertSerialId) { + latestRootCaCertSerialId = tmpRootCaCertSerailId; + } + } getLogger().info("Added certificate from file:{}.", file.getAbsolutePath()); } else { @@ -164,6 +170,9 @@ private void loadAllCertificates() { if (latestCaCertSerailId != -1) { caCertId = Long.toString(latestCaCertSerailId); } + if (latestRootCaCertSerialId != -1) { + rootCaCertId = Long.toString(latestRootCaCertSerialId); + } } } } @@ -282,7 +291,8 @@ private X509Certificate getCertificateFromScm(String certId) getLogger().info("Getting certificate with certSerialId:{}.", certId); try { - SCMSecurityProtocol scmSecurityProtocolClient = getScmSecurityClient( + SCMSecurityProtocol scmSecurityProtocolClient = + HddsServerUtil.getScmSecurityClient( (OzoneConfiguration) securityConfig.getConfiguration()); String pemEncodedCert = scmSecurityProtocolClient.getCertificate(certId); @@ -471,6 +481,8 @@ public CertificateSignRequest.Builder getCSRBuilder() builder.addIpAddress(ip.getHostAddress()); if(validator.isValid(ip.getCanonicalHostName())) { builder.addDnsName(ip.getCanonicalHostName()); + } else { + getLogger().error("InValid domain", ip.getCanonicalHostName()); } }); } catch (IOException e) { @@ -818,29 +830,63 @@ public Logger getLogger() { return logger; } - /** - * Create a scm security client, used to get SCM signed certificate. - * - * @return {@link SCMSecurityProtocol} - */ - private static SCMSecurityProtocol getScmSecurityClient( - OzoneConfiguration conf) throws IOException { - RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class, - ProtobufRpcEngine.class); - long scmVersion = - RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class); - InetSocketAddress scmSecurityProtoAdd = - HddsServerUtil.getScmAddressForSecurityProtocol(conf); - SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient = - new SCMSecurityProtocolClientSideTranslatorPB( - RPC.getProxy(SCMSecurityProtocolPB.class, scmVersion, - scmSecurityProtoAdd, UserGroupInformation.getCurrentUser(), - conf, NetUtils.getDefaultSocketFactory(conf), - Client.getRpcTimeout(conf))); - return scmSecurityClient; + public String getComponentName() { + return null; } - public String getComponentName() { + @Override + public X509Certificate getRootCACertificate() { + if (rootCaCertId != null) { + return certificateMap.get(rootCaCertId); + } return null; } + + @Override + public void storeRootCACertificate(String pemEncodedCert, boolean force) + throws CertificateException { + CertificateCodec certificateCodec = new CertificateCodec(securityConfig, + component); + try { + Path basePath = securityConfig.getCertificateLocation(component); + + X509Certificate cert = + CertificateCodec.getX509Certificate(pemEncodedCert); + String certName = String.format(CERT_FILE_NAME_FORMAT, + cert.getSerialNumber().toString()); + + certName = ROOT_CA_CERT_PREFIX + certName; + rootCaCertId = cert.getSerialNumber().toString(); + + certificateCodec.writeCertificate(basePath, certName, + pemEncodedCert, force); + certificateMap.putIfAbsent(cert.getSerialNumber().toString(), cert); + } catch (IOException | java.security.cert.CertificateException e) { + throw new CertificateException("Error while storing Root CA " + + "certificate.", e, CERTIFICATE_ERROR); + } + } + + @Override + public List listCA() throws IOException { + if (pemEncodedCACerts == null) { + updateCAList(); + } + return pemEncodedCACerts; + } + + public List updateCAList() throws IOException { + try { + SCMSecurityProtocol scmSecurityProtocolClient = + HddsServerUtil.getScmSecurityClient( + securityConfig.getConfiguration()); + pemEncodedCACerts = + scmSecurityProtocolClient.listCACertificate(); + return pemEncodedCACerts; + } catch (Exception e) { + getLogger().error("Error during updating CA list", e); + throw new CertificateException("Error during updating CA list", e, + CERTIFICATE_ERROR); + } + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java index 6bf85cac3a4b..bf1578731626 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java @@ -18,7 +18,11 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import com.google.protobuf.ServiceException; +import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; +import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; +import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.AddSCMRequest; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmInfo; @@ -27,6 +31,9 @@ import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.proxy.SCMBlockLocationFailoverProxyProvider; +import org.apache.hadoop.hdds.security.exception.SCMSecurityException; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.db.DBDefinition; import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition; @@ -35,6 +42,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneSecurityUtil; +import org.apache.hadoop.util.Time; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.FileUtils; import org.slf4j.Logger; @@ -47,7 +55,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath; @@ -326,4 +336,123 @@ public static void checkSecurityAndSCMHAEnabled(OzoneConfiguration conf) { } } } + + /** + * Build CA list which need to be passed to client. + * + * If certificate client is null, obtain the list of CA using SCM security + * client, else it uses certificate client. + * @param certClient + * @param configuration + * @return list of CA + * @throws IOException + */ + public static List buildCAList(CertificateClient certClient, + ConfigurationSource configuration) throws IOException { + //TODO: make it configurable. + long waitTime = 5 * 60 * 1000L; + long retryTime = 10 * 1000L; + long currentTime = Time.monotonicNow(); + List caCertPemList = null; + if (certClient != null) { + caCertPemList = new ArrayList<>(); + if (!SCMHAUtils.isSCMHAEnabled(configuration)) { + if (certClient.getRootCACertificate() != null) { + caCertPemList.add(CertificateCodec.getPEMEncodedString( + certClient.getRootCACertificate())); + } + caCertPemList.add(CertificateCodec.getPEMEncodedString( + certClient.getCACertificate())); + } else { + // TODO: If SCMs are bootstrapped later, then listCA need to be + // refetched if listCA size is less than scm ha config node list size. + // For now when Client of SCM's are started we compare their node list + // size and ca list size if it is as expected, we return the ca list. + boolean caListUpToDate; + Collection scmNodes = SCMHAUtils.getSCMNodeIds(configuration); + // TODO: make them configurable. + if (scmNodes.size() > 1) { + do { + caCertPemList = certClient.updateCAList(); + caListUpToDate = + caCertPemList.size() == scmNodes.size() + 1 ? true : false; + if (!caListUpToDate) { + try { + Thread.sleep(retryTime); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + } while (!caListUpToDate && + Time.monotonicNow() - currentTime < waitTime); + checkCertCount(caCertPemList.size(), scmNodes.size() + 1); + } else { + caCertPemList = certClient.updateCAList(); + } + } + } else { + if (!SCMHAUtils.isSCMHAEnabled(configuration)) { + caCertPemList = new ArrayList<>(); + SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient = + HddsServerUtil.getScmSecurityClient(configuration); + SCMGetCertResponseProto scmGetCertResponseProto = + scmSecurityProtocolClient.getCACert(); + if (scmGetCertResponseProto.hasX509Certificate()) { + caCertPemList.add(scmGetCertResponseProto.getX509Certificate()); + } + if (scmGetCertResponseProto.hasX509RootCACertificate()) { + caCertPemList.add(scmGetCertResponseProto.getX509RootCACertificate()); + } + } else { + Collection scmNodes = SCMHAUtils.getSCMNodeIds(configuration); + SCMSecurityProtocol scmSecurityProtocolClient = + HddsServerUtil.getScmSecurityClient(configuration); + boolean caListUpToDate; + if (scmNodes.size() > 1) { + do { + caCertPemList = + scmSecurityProtocolClient.listCACertificate(); + caListUpToDate = + caCertPemList.size() == scmNodes.size() + 1 ? true : false; + if (!caListUpToDate) { + try { + Thread.sleep(retryTime); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + } while (!caListUpToDate && + Time.monotonicNow() - currentTime < waitTime); + checkCertCount(caCertPemList.size(), scmNodes.size() + 1); + } else { + caCertPemList = scmSecurityProtocolClient.listCACertificate(); + } + } + } + return caCertPemList; + } + + /** + * Build CA list which need to be passed to client. + * + * @param configuration + * @return list of CA + * @throws IOException + */ + public static List buildCAList(ConfigurationSource configuration) + throws IOException { + return buildCAList(null, configuration); + } + + private static void checkCertCount(int certCount, int expectedCount) + throws SCMSecurityException{ + if (certCount != expectedCount) { + LOG.error("Unable to obtain CA list for SCM cluster, obtained CA list " + + "size is {}, where as expected list size is {}", + certCount, expectedCount); + throw new SCMSecurityException("Unable to obtain complete CA list"); + } + } + + } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index ddc7e04b0652..8539910067a6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -431,7 +431,7 @@ public static String getDatanodeIdFilePath(ConfigurationSource conf) { * @throws IOException */ public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient( - OzoneConfiguration conf) throws IOException { + ConfigurationSource conf) throws IOException { return new SCMSecurityProtocolClientSideTranslatorPB( new SCMSecurityProtocolFailoverProxyProvider(conf, UserGroupInformation.getCurrentUser())); diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java index c60c9753555e..3da7e521aea0 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java @@ -44,7 +44,11 @@ public void storeValidCertificate(BigInteger serialID, @Override public void checkValidCertID(BigInteger serialID) throws IOException { + } + @Override + public void storeValidScmCertificate(BigInteger serialID, + X509Certificate certificate) throws IOException { } @Override diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java index 9b917dbfac3b..a673808e77b2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java @@ -20,10 +20,10 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmNodeDetailsProto; import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.server.SCMCertStore; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer; +import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore; import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer; import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultCAProfile; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; @@ -32,6 +32,9 @@ import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.utils.HddsServerUtil; +import org.apache.ratis.conf.Parameters; +import org.apache.ratis.grpc.GrpcConfigKeys; +import org.apache.ratis.grpc.GrpcTlsConfig; import org.bouncycastle.cert.X509CertificateHolder; import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; @@ -216,7 +219,7 @@ private static void getPrimarySCMSelfSignedCert(CertificateClient client, * @param scmStorageConfig */ public static CertificateServer initializeRootCertificateServer( - OzoneConfiguration config, SCMCertStore scmCertStore, + OzoneConfiguration config, CertificateStore scmCertStore, SCMStorageConfig scmStorageConfig) throws IOException { String subject = SCM_ROOT_CA_PREFIX + @@ -281,4 +284,26 @@ private static void persistSubCACertificate(OzoneConfiguration config, certCodec.writeCertificate(certificateHolder); } + /** + * Create Server TLS parameters required for Ratis Server. + * @param conf + * @param caClient + * @return + */ + public static Parameters createServerTlsParameters(SecurityConfig conf, + CertificateClient caClient) { + Parameters parameters = new Parameters(); + + if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { + GrpcTlsConfig config = new GrpcTlsConfig( + caClient.getPrivateKey(), caClient.getCertificate(), + caClient.getCACertificate(), true); + GrpcConfigKeys.Server.setTlsConf(parameters, config); + GrpcConfigKeys.Admin.setTlsConf(parameters, config); + GrpcConfigKeys.Client.setTlsConf(parameters, config); + GrpcConfigKeys.TLS.setConf(parameters, config); + } + + return parameters; + } } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java index 4835d033fcf8..0702a92f7c0c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java @@ -344,7 +344,10 @@ public void startServices() throws IOException { scm.getScmBlockManager().getDeletedBlockLog().reinitialize( metadataStore.getDeletedBlocksTXTable()); if (OzoneSecurityUtil.isSecurityEnabled(conf)) { - scm.getCertificateServer().reinitialize(metadataStore); + if (scm.getRootCertificateServer() != null) { + scm.getRootCertificateServer().reinitialize(metadataStore); + } + scm.getScmCertificateServer().reinitialize(metadataStore); } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java index 79da583f4255..c30ba7059e25 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java @@ -33,8 +33,10 @@ import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType; import org.apache.hadoop.hdds.scm.AddSCMRequest; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.util.Time; +import org.apache.ratis.conf.Parameters; import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.RaftClientReply; @@ -81,9 +83,14 @@ public class SCMRatisServerImpl implements SCMRatisServer { // persisted in the raft log post leader election. Now, when the primary // scm boots up, it has peer info embedded in the raft log and will // trigger leader election. - this.server = - newRaftServer(scm.getScmId(), conf).setStateMachine(stateMachine) - .setGroup(RaftGroup.valueOf(groupId)).build(); + + Parameters parameters = + HASecurityUtils.createServerTlsParameters(new SecurityConfig(conf), + scm.getScmCertificateClient()); + this.server = newRaftServer(scm.getScmId(), conf) + .setStateMachine(stateMachine) + .setGroup(RaftGroup.valueOf(groupId)) + .setParameters(parameters).build(); this.division = server.getDivision(groupId); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java index cc0c776c4180..8a71eb5c2b8e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java @@ -170,7 +170,8 @@ public SCMGetCertResponseProto getDataNodeCertificate( .newBuilder() .setResponseCode(ResponseCode.success) .setX509Certificate(certificate) - .setX509CACertificate(impl.getCACertificate()); + .setX509CACertificate(impl.getCACertificate()) + .setX509RootCACertificate(impl.getRootCACertificate()); return builder.build(); @@ -194,7 +195,8 @@ public SCMGetCertResponseProto getSCMCertificate( .newBuilder() .setResponseCode(ResponseCode.success) .setX509Certificate(certificate) - .setX509CACertificate(impl.getCACertificate()); + .setX509CACertificate(impl.getRootCACertificate()) + .setX509RootCACertificate(impl.getRootCACertificate()); return builder.build(); @@ -216,7 +218,8 @@ public SCMGetCertResponseProto getOMCertificate( .newBuilder() .setResponseCode(ResponseCode.success) .setX509Certificate(certificate) - .setX509CACertificate(impl.getCACertificate()); + .setX509CACertificate(impl.getCACertificate()) + .setX509RootCACertificate(impl.getRootCACertificate()); return builder.build(); } @@ -243,7 +246,9 @@ public SCMGetCertResponseProto getCACertificate( SCMGetCertResponseProto .newBuilder() .setResponseCode(ResponseCode.success) - .setX509Certificate(certificate); + .setX509Certificate(certificate) + .setX509CACertificate(certificate) + .setX509RootCACertificate(impl.getRootCACertificate()); return builder.build(); } @@ -271,6 +276,7 @@ public SCMGetCertResponseProto getRootCACertificate() throws IOException { SCMGetCertResponseProto .newBuilder() .setResponseCode(ResponseCode.success) + .setX509Certificate(rootCACertificate) .setX509RootCACertificate(rootCACertificate); return builder.build(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java index ee4bc20387e0..e76d200b35d2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java @@ -101,7 +101,7 @@ public void storeValidCertificate(BigInteger serialID, * @param certificate - Certificate to persist. * @throws IOException - on Failure. */ - private void storeValidScmCertificate(BigInteger serialID, + public void storeValidScmCertificate(BigInteger serialID, X509Certificate certificate) throws IOException { lock.lock(); try { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java index 5a1ecf3ff127..8ca5cebb704e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java @@ -57,7 +57,6 @@ import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.CERTIFICATE_NOT_FOUND; import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CA_CERT_FAILED; import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CERTIFICATE_FAILED; -import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_ROOT_CA_CERT_FAILED; import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateApprover.ApprovalType.KERBEROS_TRUSTED; /** @@ -70,7 +69,7 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol { private static final Logger LOGGER = LoggerFactory .getLogger(SCMSecurityProtocolServer.class); - private final CertificateServer certificateServer; + private final CertificateServer rootCertificateServer; private final RPC.Server rpcServer; private final InetSocketAddress rpcAddress; private final ProtocolMessageMetrics metrics; @@ -80,7 +79,7 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol { CertificateServer certificateServer, StorageContainerManager scm) throws IOException { this.storageContainerManager = scm; - this.certificateServer = certificateServer; + this.rootCertificateServer = certificateServer; final int handlerCount = conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY, ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT); @@ -159,7 +158,7 @@ public String getSCMCertificate(ScmNodeDetailsProto scmNodeDetails, scmNodeDetails.getHostName(), scmNodeDetails.getScmNodeId()); // Check clusterID - if (storageContainerManager.getClusterId().equals( + if (!storageContainerManager.getClusterId().equals( scmNodeDetails.getClusterId())) { throw new IOException("SCM ClusterId mismatch. Peer SCM ClusterId " + scmNodeDetails.getClusterId() + ", primary SCM ClusterId " @@ -179,9 +178,15 @@ public String getSCMCertificate(ScmNodeDetailsProto scmNodeDetails, */ private String getEncodedCertToString(String certSignReq, NodeType nodeType) throws IOException { - Future future = - certificateServer.requestCertificate(certSignReq, - KERBEROS_TRUSTED, nodeType); + + Future future; + if (nodeType == NodeType.SCM) { + future = rootCertificateServer.requestCertificate(certSignReq, + KERBEROS_TRUSTED, nodeType); + } else { + future = storageContainerManager.getScmCertificateServer() + .requestCertificate(certSignReq, KERBEROS_TRUSTED, nodeType); + } try { return CertificateCodec.getPEMEncodedString(future.get()); } catch (InterruptedException e) { @@ -226,7 +231,7 @@ public String getCertificate(String certSerialId) throws IOException { certSerialId); try { X509Certificate certificate = - certificateServer.getCertificate(certSerialId); + rootCertificateServer.getCertificate(certSerialId); if (certificate != null) { return CertificateCodec.getPEMEncodedString(certificate); } @@ -249,7 +254,7 @@ public String getCACertificate() throws IOException { LOGGER.debug("Getting CA certificate."); try { return CertificateCodec.getPEMEncodedString( - certificateServer.getCACertificate()); + storageContainerManager.getScmCertificateServer().getCACertificate()); } catch (CertificateException e) { throw new SCMSecurityException("getRootCertificate operation failed. ", e, GET_CA_CERT_FAILED); @@ -269,8 +274,8 @@ public String getCACertificate() throws IOException { public List listCertificate(NodeType role, long startSerialId, int count, boolean isRevoked) throws IOException { List certificates = - certificateServer.listCertificate(role, startSerialId, count, - isRevoked); + storageContainerManager.getScmCertificateServer() + .listCertificate(role, startSerialId, count, isRevoked); List results = new ArrayList<>(certificates.size()); for (X509Certificate cert : certificates) { try { @@ -288,22 +293,18 @@ public List listCertificate(NodeType role, public List listCACertificate() throws IOException { List caCerts = listCertificate(NodeType.SCM, 0, 10, false); - caCerts.add(getRootCACertificate()); return caCerts; } @Override public String getRootCACertificate() throws IOException { LOGGER.debug("Getting Root CA certificate."); - //TODO: This code will be modified after HDDS-4897 is merged and - // integrated. For now getting RootCA cert from certificateServer. - try { + if (storageContainerManager.getScmStorageConfig() + .getPrimaryScmNodeId() != null) { return CertificateCodec.getPEMEncodedString( - certificateServer.getCACertificate()); - } catch (CertificateException e) { - throw new SCMSecurityException("getRootCertificate operation failed. ", - e, GET_ROOT_CA_CERT_FAILED); + storageContainerManager.getScmCertificateClient().getCACertificate()); } + return null; } public RPC.Server getRpcServer() { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStarterInterface.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStarterInterface.java index 1ae29b1a7c35..f037be60b1c3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStarterInterface.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStarterInterface.java @@ -22,6 +22,8 @@ package org.apache.hadoop.hdds.scm.server; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.security.authentication.client.AuthenticationException; + import java.io.IOException; /** @@ -34,6 +36,6 @@ public interface SCMStarterInterface { boolean init(OzoneConfiguration conf, String clusterId) throws IOException; boolean bootStrap(OzoneConfiguration conf) - throws IOException; + throws IOException, AuthenticationException; String generateClusterId(); } \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java index 5d9f4de92783..72d252503d8c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java @@ -27,6 +27,7 @@ import java.util.Properties; import java.util.UUID; +import static org.apache.hadoop.ozone.OzoneConsts.PRIMARY_SCM_NODE_ID; import static org.apache.hadoop.ozone.OzoneConsts.SCM_CERT_SERIAL_ID; import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID; import static org.apache.hadoop.ozone.OzoneConsts.STORAGE_DIR; @@ -94,4 +95,22 @@ public void setScmCertSerialId(String certSerialId) throws IOException { public String getScmCertSerialId() { return getStorageInfo().getProperty(SCM_CERT_SERIAL_ID); } + + /** + * Set primary SCM node ID. + * @param scmId + * @throws IOException + */ + public void setPrimaryScmNodeId(String scmId) throws IOException { + getStorageInfo().setProperty(PRIMARY_SCM_NODE_ID, scmId); + + } + + /** + * Retrieves the primary SCM node ID from the version file. + * @return Primary SCM node ID. + */ + public String getPrimaryScmNodeId() { + return getStorageInfo().getProperty(PRIMARY_SCM_NODE_ID); + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 22d41fd1a9ef..510261e3ce8a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -23,6 +23,7 @@ import javax.management.ObjectName; import java.io.IOException; +import java.math.BigInteger; import java.net.InetAddress; import java.net.InetSocketAddress; import com.google.common.annotations.VisibleForTesting; @@ -36,6 +37,7 @@ import java.security.cert.X509Certificate; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.UUID; @@ -54,10 +56,12 @@ import org.apache.hadoop.hdds.scm.PlacementPolicy; import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl; import org.apache.hadoop.hdds.scm.container.ContainerManagerV2; +import org.apache.hadoop.hdds.scm.ha.HASecurityUtils; import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.ha.SCMHAManager; import org.apache.hadoop.hdds.scm.ha.SCMHAManagerImpl; import org.apache.hadoop.hdds.scm.ha.SCMHANodeDetails; +import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; import org.apache.hadoop.hdds.scm.ha.SCMServiceManager; import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails; import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl; @@ -66,6 +70,8 @@ import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore; import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile; +import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.scm.ScmConfig; @@ -108,11 +114,9 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; import org.apache.hadoop.hdds.scm.pipeline.choose.algorithms.PipelineChoosePolicyFactory; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; -import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer; import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer; -import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; @@ -122,6 +126,7 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.util.MBeans; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.apache.hadoop.ozone.common.Storage.StorageState; @@ -132,16 +137,17 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.util.JvmPauseMonitor; -import org.apache.ratis.grpc.GrpcTlsConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT_DEFAULT; import static org.apache.hadoop.hdds.utils.HAUtils.checkSecurityAndSCMHAEnabled; +import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY; import static org.apache.hadoop.ozone.OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME; import static org.apache.hadoop.ozone.OzoneConsts.SCM_ROOT_CA_PREFIX; +import static org.apache.hadoop.ozone.OzoneConsts.SCM_SUB_CA_PREFIX; /** * StorageContainerManager is the main entry point for the service that @@ -186,6 +192,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private NodeDecommissionManager scmDecommissionManager; private SCMMetadataStore scmMetadataStore; + private CertificateStore certificateStore; private SCMHAManager scmHAManager; private SCMContext scmContext; private SequenceIdGenerator sequenceIdGen; @@ -215,14 +222,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private final LeaseManager commandWatcherLeaseManager; private SCMSafeModeManager scmSafeModeManager; - private CertificateServer certificateServer; - private GrpcTlsConfig grpcTlsConfig; + private CertificateServer rootCertificateServer; + private CertificateServer scmCertificateServer; + private SCMCertificateClient scmCertificateClient; private JvmPauseMonitor jvmPauseMonitor; private final OzoneConfiguration configuration; private SCMContainerMetrics scmContainerMetrics; private SCMContainerPlacementMetrics placementMetrics; private MetricsSystem ms; + private String primaryScmNodeId; /** * Network topology Map. @@ -287,6 +296,9 @@ private StorageContainerManager(OzoneConfiguration conf, "failure.", ResultCodes.SCM_NOT_INITIALIZED); } + primaryScmNodeId = scmStorageConfig.getPrimaryScmNodeId(); + initializeCertificateClient(); + /** * Important : This initialization sequence is assumed by some of our tests. * The testSecureOzoneCluster assumes that security checks have to be @@ -294,7 +306,7 @@ private StorageContainerManager(OzoneConfiguration conf, * add any other initialization above the Security checks please. */ if (OzoneSecurityUtil.isSecurityEnabled(conf)) { - loginAsSCMUser(conf); + loginAsSCMUserIfSecurityEnabled(scmHANodeDetails, conf); } // Creates the SCM DBs or opens them if it exists. @@ -319,7 +331,8 @@ private StorageContainerManager(OzoneConfiguration conf, // if no Security, we do not create a Certificate Server at all. // This allows user to boot SCM without security temporarily // and then come back and enable it without any impact. - certificateServer = null; + rootCertificateServer = null; + scmCertificateServer = null; securityProtocolServer = null; } @@ -400,6 +413,14 @@ private StorageContainerManager(OzoneConfiguration conf, registerMetricsSource(this); } + private void initializeCertificateClient() { + if (primaryScmNodeId != null) { + scmCertificateClient = new SCMCertificateClient( + new SecurityConfig(configuration), + scmStorageConfig.getScmCertSerialId()); + } + } + public OzoneConfiguration getConfiguration() { return configuration; } @@ -558,44 +579,105 @@ private void initializeSystemManagers(OzoneConfiguration conf, */ private void initializeCAnSecurityProtocol(OzoneConfiguration conf, SCMConfigurator configurator) throws IOException { - if(configurator.getCertificateServer() != null) { - this.certificateServer = configurator.getCertificateServer(); + + // TODO: Support Certificate Server loading via Class Name loader. + // So it is easy to use different Certificate Servers if needed. + if(this.scmMetadataStore == null) { + LOG.error("Cannot initialize Certificate Server without a valid meta " + + "data layer."); + throw new SCMException("Cannot initialize CA without a valid metadata " + + "store", ResultCodes.SCM_NOT_INITIALIZED); + } + + certificateStore = + new SCMCertStore.Builder().setMetadaStore(scmMetadataStore) + .setRatisServer(scmHAManager.getRatisServer()) + .setCRLSequenceId(getLastSequenceIdForCRL()).build(); + + + // If primary SCM node Id is set it means this is a cluster which has + // performed init with SCM HA version code. + if (primaryScmNodeId != null) { + // Start specific instance SCM CA server. + String subject = SCM_SUB_CA_PREFIX + + InetAddress.getLocalHost().getHostName(); + if (configurator.getCertificateServer() != null) { + this.scmCertificateServer = configurator.getCertificateServer(); + } else { + scmCertificateServer = new DefaultCAServer(subject, + scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(), + certificateStore, new DefaultProfile(), + scmCertificateClient.getComponentName()); + // INTERMEDIARY_CA which issues certs to DN and OM. + scmCertificateServer.init(new SecurityConfig(configuration), + CertificateServer.CAType.INTERMEDIARY_CA); + } + + if (primaryScmNodeId.equals(scmStorageConfig.getScmId())) { + if (configurator.getCertificateServer() != null) { + this.rootCertificateServer = configurator.getCertificateServer(); + } else { + rootCertificateServer = + HASecurityUtils.initializeRootCertificateServer( + conf, certificateStore, scmStorageConfig); + rootCertificateServer.init(new SecurityConfig(configuration), + CertificateServer.CAType.SELF_SIGNED_CA); + } + + BigInteger certSerial = + scmCertificateClient.getCertificate().getSerialNumber(); + // Store the certificate in DB. On primary SCM when init happens, the + // certificate is not persisted to DB. As we don't have Metadatstore + // and ratis server initialized with statemachine. We need to do only + // for primary scm, for other bootstrapped scm's certificates will be + // persisted via ratis. + if (certificateStore.getCertificateByID(certSerial, + VALID_CERTS) == null) { + LOG.info("Storing certSerial {}", certSerial); + certificateStore.storeValidScmCertificate( + certSerial, scmCertificateClient.getCertificate()); + } + X509Certificate rootCACert = scmCertificateClient.getCACertificate(); + if (certificateStore.getCertificateByID(rootCACert.getSerialNumber(), + VALID_CERTS) == null) { + LOG.info("Storing root certSerial {}", rootCACert.getSerialNumber()); + certificateStore.storeValidScmCertificate( + rootCACert.getSerialNumber(), rootCACert); + } + } else { + rootCertificateServer = null; + } } else { - // This assumes that SCM init has run, and DB metadata stores are created. - certificateServer = initializeCertificateServer( + // On a upgraded cluster primary scm nodeId will not be set as init will + // not be run again after upgrade. So for a upgraded cluster where init + // has not happened again we will have setup like before where it has + // one CA server which is issuing certificates to DN and OM. + String subject = SCM_ROOT_CA_PREFIX + + InetAddress.getLocalHost().getHostName(); + rootCertificateServer = new DefaultCAServer(subject, getScmStorageConfig().getClusterID(), - getScmStorageConfig().getScmId()); + getScmStorageConfig().getScmId(), certificateStore, + new DefaultProfile(), + SCM_ROOT_CA_COMPONENT_NAME); + scmCertificateServer = rootCertificateServer; } - // TODO: Support Intermediary CAs in future. - certificateServer.init(new SecurityConfig(conf), - CertificateServer.CAType.SELF_SIGNED_CA); + securityProtocolServer = new SCMSecurityProtocolServer(conf, - certificateServer, this); + rootCertificateServer, this); + } - grpcTlsConfig = createTlsClientConfigForSCM(new SecurityConfig(conf), - certificateServer); + public CertificateServer getRootCertificateServer() { + return rootCertificateServer; } - public CertificateServer getCertificateServer() { - return certificateServer; + public CertificateServer getScmCertificateServer() { + return scmCertificateServer; } - // For Internal gRPC client from SCM to DN with gRPC TLS - static GrpcTlsConfig createTlsClientConfigForSCM(SecurityConfig conf, - CertificateServer certificateServer) throws IOException { - if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { - try { - X509Certificate caCert = - CertificateCodec.getX509Certificate( - certificateServer.getCACertificate()); - return new GrpcTlsConfig(null, null, - caCert, false); - } catch (CertificateException ex) { - throw new SCMSecurityException("Fail to find SCM CA certificate.", ex); - } - } - return null; + public SCMCertificateClient getScmCertificateClient() { + return scmCertificateClient; } + /** * Init the metadata store based on the configurator. * @param conf - Config @@ -617,33 +699,35 @@ private void initalizeMetadataStore(OzoneConfiguration conf, * * @param conf */ - private void loginAsSCMUser(ConfigurationSource conf) + private static void loginAsSCMUserIfSecurityEnabled( + SCMHANodeDetails scmhaNodeDetails, ConfigurationSource conf) throws IOException, AuthenticationException { - if (LOG.isDebugEnabled()) { - ScmConfig scmConfig = configuration.getObject(ScmConfig.class); - LOG.debug("Ozone security is enabled. Attempting login for SCM user. " - + "Principal: {}, keytab: {}", - scmConfig.getKerberosPrincipal(), - scmConfig.getKerberosKeytab()); - } + if (OzoneSecurityUtil.isSecurityEnabled(conf)) { + if (LOG.isDebugEnabled()) { + ScmConfig scmConfig = conf.getObject(ScmConfig.class); + LOG.debug("Ozone security is enabled. Attempting login for SCM user. " + + "Principal: {}, keytab: {}", + scmConfig.getKerberosPrincipal(), + scmConfig.getKerberosKeytab()); + } - Configuration hadoopConf = - LegacyHadoopConfigurationSource.asHadoopConfiguration(conf); - if (SecurityUtil.getAuthenticationMethod(hadoopConf).equals( - AuthenticationMethod.KERBEROS)) { - UserGroupInformation.setConfiguration(hadoopConf); - InetSocketAddress socAddr = HddsServerUtil - .getScmBlockClientBindAddress(conf); - SecurityUtil.login(hadoopConf, + Configuration hadoopConf = + LegacyHadoopConfigurationSource.asHadoopConfiguration(conf); + if (SecurityUtil.getAuthenticationMethod(hadoopConf).equals( + AuthenticationMethod.KERBEROS)) { + UserGroupInformation.setConfiguration(hadoopConf); + InetSocketAddress socketAddress = getScmAddress(scmhaNodeDetails, conf); + SecurityUtil.login(hadoopConf, ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY, - socAddr.getHostName()); - } else { - throw new AuthenticationException(SecurityUtil.getAuthenticationMethod( - hadoopConf) + " authentication method not support. " - + "SCM user login failed."); + socketAddress.getHostName()); + } else { + throw new AuthenticationException(SecurityUtil.getAuthenticationMethod( + hadoopConf) + " authentication method not support. " + + "SCM user login failed."); + } + LOG.info("SCM login successful."); } - LOG.info("SCM login successful."); } /** @@ -749,14 +833,17 @@ public static RPC.Server startRpcServer( * @throws IOException if init fails due to I/O error */ public static boolean scmBootstrap(OzoneConfiguration conf) - throws IOException { + throws AuthenticationException, IOException { if (!SCMHAUtils.isSCMHAEnabled(conf)) { LOG.error("Bootstrap is not supported without SCM HA."); return false; } + SCMHANodeDetails scmhaNodeDetails = SCMHANodeDetails.loadSCMHAConfig(conf); + + loginAsSCMUserIfSecurityEnabled(scmhaNodeDetails, conf); // The node here will try to fetch the cluster id from any of existing // running SCM instances. - SCMHANodeDetails scmhaNodeDetails = SCMHANodeDetails.loadSCMHAConfig(conf); + String primordialSCM = SCMHAUtils.getPrimordialSCM(conf); String selfNodeId = scmhaNodeDetails.getLocalNodeDetails().getNodeId(); if (primordialSCM != null && SCMHAUtils.isPrimordialSCM(conf, selfNodeId)) { @@ -793,7 +880,18 @@ public static boolean scmBootstrap(OzoneConfiguration conf) // SCM Node info containing hostname to scm Id mappings // will be persisted into the version file once this node gets added // to existing SCM ring post node regular start up. + + if(OzoneSecurityUtil.isSecurityEnabled(conf)) { + HASecurityUtils.initializeSecurity(scmStorageConfig, + scmInfo.getScmId(), conf, getScmAddress(scmhaNodeDetails, conf), + false); + } + scmStorageConfig.setPrimaryScmNodeId(scmInfo.getScmId()); scmStorageConfig.initialize(); + LOG.info("SCM BootStrap is successful for ClusterID {}, SCMID {}", + scmInfo.getClusterId(), scmStorageConfig.getScmId()); + LOG.info("Primary SCM Node ID {}", + scmStorageConfig.getPrimaryScmNodeId()); } catch (IOException ioe) { LOG.error("Could not initialize SCM version file", ioe); return false; @@ -832,12 +930,20 @@ public static boolean scmInit(OzoneConfiguration conf, Preconditions.checkNotNull(UUID.fromString(clusterId)); scmStorageConfig.setClusterId(clusterId); } - scmStorageConfig.initialize(); + if (SCMHAUtils.isSCMHAEnabled(conf)) { SCMRatisServerImpl.initialize(scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(), haDetails.getLocalNodeDetails(), conf); } + + if (OzoneSecurityUtil.isSecurityEnabled(conf)) { + HASecurityUtils.initializeSecurity(scmStorageConfig, + scmStorageConfig.getScmId(), conf, getScmAddress(haDetails, + conf), true); + } + scmStorageConfig.setPrimaryScmNodeId(scmStorageConfig.getScmId()); + scmStorageConfig.initialize(); LOG.info("SCM initialization succeeded. Current cluster id for sd={}" + "; cid={}; layoutVersion={}; scmId={}", scmStorageConfig.getStorageDir(), scmStorageConfig.getClusterID(), @@ -860,6 +966,40 @@ public static boolean scmInit(OzoneConfiguration conf, } } + private static InetSocketAddress getScmAddress(SCMHANodeDetails haDetails, + ConfigurationSource conf) throws IOException { + List scmNodeInfoList = SCMNodeInfo.buildNodeInfo( + conf); + Preconditions.checkNotNull(scmNodeInfoList, "scmNodeInfoList is null"); + + InetSocketAddress scmAddress = null; + if (SCMHAUtils.getScmServiceId(conf) != null) { + for (SCMNodeInfo scmNodeInfo : scmNodeInfoList) { + if (haDetails.getLocalNodeDetails().getNodeId() != null + && haDetails.getLocalNodeDetails().getNodeId().equals( + scmNodeInfo.getNodeId())) { + scmAddress = + NetUtils.createSocketAddr(scmNodeInfo.getBlockClientAddress()); + } + } + } else { + // Get Local host and use scm client port + if (scmNodeInfoList.get(0).getBlockClientAddress() == null) { + LOG.error("SCM Address not able to figure out from config, finding " + + "hostname from InetAddress."); + scmAddress = + NetUtils.createSocketAddr(InetAddress.getLocalHost().getHostName(), + ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT); + } else { + scmAddress = NetUtils.createSocketAddr( + scmNodeInfoList.get(0).getBlockClientAddress()); + } + } + + + return scmAddress; + } + /** * Initialize SCM metrics. */ @@ -1019,6 +1159,7 @@ public void start() throws IOException { getDatanodeProtocolServer().start(); if (getSecurityProtocolServer() != null) { getSecurityProtocolServer().start(); + persistSCMCertificates(); } scmBlockManager.start(); @@ -1039,6 +1180,35 @@ public void start() throws IOException { setStartTime(); } + private void persistSCMCertificates() throws IOException { + // Fetch all CA's and persist during startup on bootstrap nodes. This + // is primarily being done to persist primary SCM Cert and Root CA. + // TODO: see if we can avoid doing this during every restart. + if (primaryScmNodeId != null && !primaryScmNodeId.equals( + scmStorageConfig.getScmId())) { + List pemEncodedCerts = + scmCertificateClient.updateCAList(); + + // Write the primary SCM CA and Root CA during startup. + for (String cert : pemEncodedCerts) { + try { + X509Certificate x509Certificate = + CertificateCodec.getX509Certificate(cert); + if (certificateStore.getCertificateByID( + x509Certificate.getSerialNumber(), VALID_CERTS) == null) { + LOG.info("Persist certserial {} on Scm Bootstrap Node {}", + x509Certificate.getSerialNumber(), scmStorageConfig.getScmId()); + certificateStore.storeValidScmCertificate( + x509Certificate.getSerialNumber(), x509Certificate); + } + } catch (CertificateException ex) { + LOG.error("Error while decoding CA Certificate", ex); + throw new IOException(ex); + } + } + } + } + /** * Stop service. */ diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java index 4c7b69348970..8960269cb445 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerStarter.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.HddsVersionInfo; import org.apache.hadoop.ozone.common.StorageInfo; +import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -174,7 +175,7 @@ public boolean init(OzoneConfiguration conf, String clusterId) @Override public boolean bootStrap(OzoneConfiguration conf) - throws IOException{ + throws AuthenticationException, IOException { return StorageContainerManager.scmBootstrap(conf); } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index aab05247b90e..7555d105e4fa 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -26,7 +26,6 @@ import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadContainerResponseProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -42,8 +41,8 @@ import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.tracing.TracingUtil; +import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.slf4j.Logger; @@ -53,7 +52,6 @@ import com.google.common.annotations.VisibleForTesting; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT; -import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClient; import static org.apache.hadoop.ozone.ClientVersions.CURRENT_VERSION; /** @@ -102,14 +100,10 @@ private XceiverClientManager newXCeiverClientManager(ConfigurationSource conf) throws IOException { XceiverClientManager manager; if (OzoneSecurityUtil.isSecurityEnabled(conf)) { - SecurityConfig securityConfig = new SecurityConfig(conf); - SCMSecurityProtocol scmSecurityProtocolClient = getScmSecurityClient( - (OzoneConfiguration) securityConfig.getConfiguration()); - String caCertificate = - scmSecurityProtocolClient.getCACertificate(); + List caCertificates = HAUtils.buildCAList(conf); manager = new XceiverClientManager(conf, conf.getObject(XceiverClientManager.ScmClientConfig.class), - caCertificate); + caCertificates); } else { manager = new XceiverClientManager(conf); } diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java index d4396818a0f8..cc84d536d0be 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java @@ -61,6 +61,7 @@ public ScmClient createScmClient() { return new ContainerOperationClient(conf); } catch (IOException ex) { + ex.printStackTrace(); throw new IllegalArgumentException("Can't create SCM client", ex); } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index ed85a32adb13..d3d93a9324bc 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -27,6 +27,7 @@ import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; @@ -173,12 +174,17 @@ public RpcClient(ConfigurationSource conf, String omServiceId) dtService = omTransport.getDelegationTokenService(); ServiceInfoEx serviceInfoEx = ozoneManagerClient.getServiceInfo(); String caCertPem = null; + List caCertPems = null; if (OzoneSecurityUtil.isSecurityEnabled(conf)) { caCertPem = serviceInfoEx.getCaCertificate(); } + caCertPems = serviceInfoEx.getCaCertPemList(); + if (caCertPems == null || caCertPems.isEmpty()) { + caCertPems = Collections.singletonList(caCertPem); + } this.xceiverClientManager = new XceiverClientManager(conf, - conf.getObject(XceiverClientManager.ScmClientConfig.class), caCertPem); + conf.getObject(XceiverClientManager.ScmClientConfig.class), caCertPems); int configuredChunkSize = (int) conf .getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfoEx.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfoEx.java index a90be635687a..e7968b890e9d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfoEx.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfoEx.java @@ -30,11 +30,13 @@ public class ServiceInfoEx { // PEM encoded string of SCM CA certificate. private String caCertificate; + private List caCertPemList; public ServiceInfoEx(List infoList, - String caCertificate) { + String caCertificate, List caCertPemList) { this.infoList = infoList; this.caCertificate = caCertificate; + this.caCertPemList = caCertPemList; } public List getServiceInfoList() { @@ -44,4 +46,8 @@ public List getServiceInfoList() { public String getCaCertificate() { return caCertificate; } + + public List getCaCertPemList() { + return caCertPemList; + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index a2b2f48434aa..43d72b90dfda 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1078,7 +1078,7 @@ public ServiceInfoEx getServiceInfo() throws IOException { resp.getServiceInfoList().stream() .map(ServiceInfo::getFromProtobuf) .collect(Collectors.toList()), - resp.getCaCertificate()); + resp.getCaCertificate(), resp.getCaCertsList()); } /** diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml index 42294612b28c..c3e816ba3733 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml @@ -48,9 +48,14 @@ services: ports: - 9864:9999 command: ["/opt/hadoop/bin/ozone","datanode"] + extra_hosts: + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" env_file: - docker-config environment: + WAITFOR: scm3.org:9865 KERBEROS_KEYTABS: dn HTTP OZONE_OPTS: networks: @@ -63,9 +68,14 @@ services: ports: - 9866:9999 command: ["/opt/hadoop/bin/ozone","datanode"] + extra_hosts: + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" env_file: - docker-config environment: + WAITFOR: scm3.org:9865 KERBEROS_KEYTABS: dn HTTP OZONE_OPTS: networks: @@ -78,9 +88,14 @@ services: ports: - 9868:9999 command: ["/opt/hadoop/bin/ozone","datanode"] + extra_hosts: + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" env_file: - docker-config environment: + WAITFOR: scm3.org:9865 KERBEROS_KEYTABS: dn HTTP OZONE_OPTS: networks: @@ -96,12 +111,17 @@ services: - 9890:9872 #- 18001:18001 environment: + WAITFOR: scm3.org:9865 ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION KERBEROS_KEYTABS: om HTTP OZONE_OPTS: env_file: - ./docker-config command: ["/opt/hadoop/bin/ozone","om"] + extra_hosts: + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" networks: ozone_net: ipv4_address: 172.25.0.111 @@ -115,12 +135,17 @@ services: - 9892:9872 #- 18002:18002 environment: + WAITFOR: scm3.org:9865 ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION KERBEROS_KEYTABS: om HTTP OZONE_OPTS: env_file: - ./docker-config command: ["/opt/hadoop/bin/ozone","om"] + extra_hosts: + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" networks: ozone_net: ipv4_address: 172.25.0.112 @@ -134,12 +159,17 @@ services: - 9894:9872 #- 18003:18003 environment: + WAITFOR: scm3.org:9865 ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION KERBEROS_KEYTABS: om HTTP OZONE_OPTS: env_file: - ./docker-config command: ["/opt/hadoop/bin/ozone","om"] + extra_hosts: + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" networks: ozone_net: ipv4_address: 172.25.0.113 @@ -159,40 +189,46 @@ services: networks: ozone_net: ipv4_address: 172.25.0.114 - recon: + scm1.org: image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - hostname: recon + hostname: scm1.org volumes: - ../..:/opt/hadoop ports: - - 9888:9888 - #- 18000:18000 + - 9990:9876 + - 9992:9860 env_file: - - ./docker-config + - docker-config environment: - KERBEROS_KEYTABS: recon HTTP + KERBEROS_KEYTABS: scm HTTP testuser testuser2 + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-3}" OZONE_OPTS: - command: ["/opt/hadoop/bin/ozone","recon"] + command: ["/opt/hadoop/bin/ozone","scm"] extra_hosts: - "om1: 172.25.0.111" - "om2: 172.25.0.112" - "om3: 172.25.0.113" + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" networks: ozone_net: - ipv4_address: 172.25.0.115 - scm: + ipv4_address: 172.25.0.116 + scm2.org: image: apache/ozone-runner:${OZONE_RUNNER_VERSION} - hostname: scm + hostname: scm2.org volumes: - ../..:/opt/hadoop ports: - - 9876:9876 - - 9860:9860 + - 9994:9876 + - 9996:9860 env_file: - docker-config environment: + WAITFOR: scm1.org:9865 KERBEROS_KEYTABS: scm HTTP testuser testuser2 - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-3}" OZONE_OPTS: command: ["/opt/hadoop/bin/ozone","scm"] @@ -200,9 +236,39 @@ services: - "om1: 172.25.0.111" - "om2: 172.25.0.112" - "om3: 172.25.0.113" + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" networks: ozone_net: - ipv4_address: 172.25.0.116 + ipv4_address: 172.25.0.117 + scm3.org: + image: apache/ozone-runner:${OZONE_RUNNER_VERSION} + hostname: scm3.org + volumes: + - ../..:/opt/hadoop + ports: + - 9998:9876 + - 10002:9860 + env_file: + - docker-config + environment: + WAITFOR: scm2.org:9865 + KERBEROS_KEYTABS: scm HTTP testuser testuser2 + ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION + OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "${OZONE_SAFEMODE_MIN_DATANODES:-3}" + OZONE_OPTS: + command: ["/opt/hadoop/bin/ozone","scm"] + extra_hosts: + - "om1: 172.25.0.111" + - "om2: 172.25.0.112" + - "om3: 172.25.0.113" + - "scm1.org: 172.25.0.116" + - "scm2.org: 172.25.0.117" + - "scm3.org: 172.25.0.118" + networks: + ozone_net: + ipv4_address: 172.25.0.118 networks: ozone_net: ipam: diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config index eeb05f9f796d..c67e5799cba7 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config @@ -27,12 +27,19 @@ OZONE-SITE.XML_ozone.om.http-address.id1.om2=om2 OZONE-SITE.XML_ozone.om.http-address.id1.om3=om3 OZONE-SITE.XML_ozone.om.ratis.enable=true +OZONE-SITE.XML_ozone.scm.service.ids=scmservice +OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3 +OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1.org +OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2.org +OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3.org +OZONE-SITE.XML_ozone.scm.ratis.enable=true +OZONE-SITE.XML_hdds.scm.ha.security.enable=true + OZONE-SITE.XML_ozone.om.volume.listall.allowed=false OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 -OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata @@ -61,7 +68,7 @@ CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1](testuser2.*) RULE:[2:$1@ CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms -OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/_HOST@EXAMPLE.COM OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab OZONE-SITE.XML_ozone.om.kerberos.principal=om/_HOST@EXAMPLE.COM OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab @@ -83,7 +90,7 @@ OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos -OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM +OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/_HOST@EXAMPLE.COM OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/_HOST@EXAMPLE.COM OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/test.sh index 9fba980e2563..dbc7a6931962 100755 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/test.sh @@ -20,17 +20,18 @@ export COMPOSE_DIR export SECURITY_ENABLED=true export OM_SERVICE_ID="id1" +export SCM=scm1.org # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" start_docker_env -execute_robot_test scm kinit.robot +execute_robot_test ${SCM} kinit.robot -execute_robot_test scm freon +execute_robot_test ${SCM} freon -execute_robot_test scm basic/links.robot +execute_robot_test ${SCM} basic/links.robot stop_docker_env diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index d0ead1baf0e5..33d1c1cbf3d3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.ha.HASecurityUtils; import org.apache.hadoop.hdds.scm.ha.SCMHANodeDetails; import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl; @@ -56,6 +57,7 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.client.CertificateClientTestImpl; import org.apache.hadoop.ozone.common.Storage; import org.apache.hadoop.ozone.om.OMStorage; @@ -302,9 +304,12 @@ public void testSCMSecurityProtocol() throws Exception { assertNotNull(scmSecurityProtocolClient); String caCert = scmSecurityProtocolClient.getCACertificate(); assertNotNull(caCert); + // Get some random certificate, used serial id 100 which will be + // unavailable as our serial id is time stamp. Serial id 1 is root CA, + // and it is persisted in DB. LambdaTestUtils.intercept(SCMSecurityException.class, "Certificate not found", - () -> scmSecurityProtocolClient.getCertificate("1")); + () -> scmSecurityProtocolClient.getCertificate("100")); // Case 2: User without Kerberos credentials should fail. ugi = UserGroupInformation.createRemoteUser("test"); @@ -336,6 +341,10 @@ private void initSCM() throws IOException { SCMStorageConfig scmStore = new SCMStorageConfig(conf); scmStore.setClusterId(clusterId); scmStore.setScmId(scmId); + HASecurityUtils.initializeSecurity(scmStore, scmId, conf, + NetUtils.createSocketAddr(InetAddress.getLocalHost().getHostName(), + OZONE_SCM_CLIENT_PORT_DEFAULT), true); + scmStore.setPrimaryScmNodeId(scmId); // writes the version file properties scmStore.initialize(); if (SCMHAUtils.isSCMHAEnabled(conf)) { @@ -728,7 +737,7 @@ public void validateCertificate(X509Certificate cert) throws Exception { X500Name x500Issuer = new JcaX509CertificateHolder(cert).getIssuer(); RDN cn = x500Issuer.getRDNs(BCStyle.CN)[0]; String hostName = InetAddress.getLocalHost().getHostName(); - String scmUser = "scm@" + hostName; + String scmUser = OzoneConsts.SCM_SUB_CA_PREFIX + hostName; assertEquals(scmUser, cn.getFirst().getValue().toString()); // Subject name should be om login user in real world but in this test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java index feaf63344bb0..0f487c668fe0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.client; +import java.io.IOException; import java.io.InputStream; import java.security.KeyPair; import java.security.PrivateKey; @@ -188,4 +189,23 @@ public String getComponentName() { return null; } + @Override + public X509Certificate getRootCACertificate() { + return x509Certificate; + } + + @Override + public void storeRootCACertificate(String pemEncodedCert, boolean force) { + + } + + @Override + public List listCA() throws IOException { + return null; + } + + @Override + public List updateCAList() throws IOException { + return null; + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java index c025ab33678b..3f7245b7369b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java @@ -52,6 +52,7 @@ import java.io.File; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -151,7 +152,7 @@ public void testCreateOzoneContainer() throws Exception { container.start(UUID.randomUUID().toString()); XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf, - caClient.getCACertificate()); + Collections.singletonList(caClient.getCACertificate())); if (blockTokenEnabled) { secretManager.start(caClient); diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 64c2eb654786..7269c9a68ff8 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1033,6 +1033,8 @@ message ServiceListResponse { // When security is enabled, return SCM CA certificate to Ozone client // to set up gRPC TLS for client to authenticate server(DN). optional string caCertificate = 3; + + repeated string caCerts = 4; } message DBUpdatesResponse { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index abc02830b063..cd2959e45211 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -261,6 +261,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl private OzoneBlockTokenSecretManager blockTokenMgr; private CertificateClient certClient; private String caCertPem = null; + private List caCertPemList = new ArrayList<>(); private static boolean testSecureOmFlag = false; private final Text omRpcAddressTxt; private final OzoneConfiguration configuration; @@ -1123,10 +1124,14 @@ public void start() throws IOException { metadataManager.start(configuration); startSecretManagerIfNecessary(); + + // Perform this to make it work with old clients. if (certClient != null) { - caCertPem = CertificateCodec.getPEMEncodedString( - certClient.getCACertificate()); + caCertPem = + CertificateCodec.getPEMEncodedString(certClient.getCACertificate()); + caCertPemList = HAUtils.buildCAList(certClient, configuration); } + // Set metrics and start metrics back ground thread metrics.setNumVolumes(metadataManager.countRowsInTable(metadataManager .getVolumeTable())); @@ -1478,6 +1483,13 @@ private static void getSCMSignedCert(CertificateClient client, String pemEncodedRootCert = response.getX509CACertificate(); client.storeCertificate(pemEncodedRootCert, true, true); client.storeCertificate(pemEncodedCert, true); + + // Store Root CA certificate if available. + if (response.hasX509RootCACertificate()) { + client.storeRootCACertificate(response.getX509RootCACertificate(), + true); + } + // Persist om cert serial id. omStore.setOmCertSerialId(CertificateCodec. getX509Certificate(pemEncodedCert).getSerialNumber().toString()); @@ -2675,7 +2687,7 @@ public List getServiceList() throws IOException { @Override public ServiceInfoEx getServiceInfo() throws IOException { - return new ServiceInfoEx(getServiceList(), caCertPem); + return new ServiceInfoEx(getServiceList(), caCertPem, caCertPemList); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java index 28a16f95ec2a..127e8e001f85 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java @@ -23,6 +23,7 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -651,9 +652,14 @@ private static Parameters createServerTlsParameters(SecurityConfig conf, Parameters parameters = new Parameters(); if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { + ArrayList< X509Certificate > listCA = new ArrayList<>(); + listCA.add(caClient.getCACertificate()); + if (caClient.getRootCACertificate() != null) { + listCA.add(caClient.getRootCACertificate()); + } GrpcTlsConfig config = new GrpcTlsConfig( caClient.getPrivateKey(), caClient.getCertificate(), - caClient.getCACertificate(), true); + listCA, true); GrpcConfigKeys.Server.setTlsConf(parameters, config); GrpcConfigKeys.Admin.setTlsConf(parameters, config); GrpcConfigKeys.Client.setTlsConf(parameters, config); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index d9547c4fe0e9..15cfe6a01766 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -461,6 +461,11 @@ private ServiceListResponse getServiceList(ServiceListRequest request) if (serviceInfoEx.getCaCertificate() != null) { resp.setCaCertificate(serviceInfoEx.getCaCertificate()); } + + for (String ca : serviceInfoEx.getCaCertPemList()) { + resp.addCaCerts(ca); + } + return resp.build(); } From c3f0e554c8b3e721d925883abb84c30333ac6215 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 22 Mar 2021 16:11:34 +0530 Subject: [PATCH 02/18] add doc and remove self id during bootstrap --- .../apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 16 +++++----------- .../certificate/client/CertificateClient.java | 10 ++++++++++ .../client/DefaultCertificateClient.java | 1 + .../hdds/scm/server/StorageContainerManager.java | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java index 601821d596f1..915e234e9353 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java @@ -56,17 +56,6 @@ public static boolean isSCMHAEnabled(ConfigurationSource conf) { ScmConfigKeys.OZONE_SCM_HA_ENABLE_DEFAULT); } - public static boolean isHAWithMultiSCMNodes(ConfigurationSource conf) { - if (isSCMHAEnabled(conf)) { - String scmServiceId = SCMHAUtils.getScmServiceId(conf); - Collection nodeIds = SCMHAUtils.getSCMNodeIds(conf, scmServiceId); - if (nodeIds.size() > 1) { - return true; - } - } - return false; - } - public static String getPrimordialSCM(ConfigurationSource conf) { return conf.get(ScmConfigKeys.OZONE_SCM_PRIMORDIAL_NODE_ID_KEY); } @@ -185,6 +174,11 @@ public static OzoneConfiguration removeSelfId( return conf; } + /** + * Get SCM Node Id list. + * @param configuration + * @return list of node ids. + */ public static Collection getSCMNodeIds( ConfigurationSource configuration) { String scmServiceId = SCMHAUtils.getScmServiceId(configuration); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java index 78e3406cc607..75bc98322527 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java @@ -234,8 +234,18 @@ enum InitResponse { void storeRootCACertificate(String pemEncodedCert, boolean force) throws CertificateException; + /** + * Return the pem encoded CA certificate list. + * @return list of pem encoded CA certificates. + * @throws IOException + */ List listCA() throws IOException; + /** + * Update and returns the pem encoded CA certificate list. + * @return list of pem encoded CA certificates. + * @throws IOException + */ List updateCAList() throws IOException; } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index fa9488b6662c..2c364d50dc81 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -875,6 +875,7 @@ public List listCA() throws IOException { return pemEncodedCACerts; } + @Override public List updateCAList() throws IOException { try { SCMSecurityProtocol scmSecurityProtocolClient = diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 510261e3ce8a..9c619cba779b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -883,7 +883,7 @@ public static boolean scmBootstrap(OzoneConfiguration conf) if(OzoneSecurityUtil.isSecurityEnabled(conf)) { HASecurityUtils.initializeSecurity(scmStorageConfig, - scmInfo.getScmId(), conf, getScmAddress(scmhaNodeDetails, conf), + scmInfo.getScmId(), config, getScmAddress(scmhaNodeDetails, conf), false); } scmStorageConfig.setPrimaryScmNodeId(scmInfo.getScmId()); From 5ab6a9f6ac39c389c7878079210a256d8d383c05 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 23 Mar 2021 08:29:31 +0530 Subject: [PATCH 03/18] fix logging error --- .../x509/certificate/client/DefaultCertificateClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index 2c364d50dc81..07a5d05e4ea0 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -482,7 +482,7 @@ public CertificateSignRequest.Builder getCSRBuilder() if(validator.isValid(ip.getCanonicalHostName())) { builder.addDnsName(ip.getCanonicalHostName()); } else { - getLogger().error("InValid domain", ip.getCanonicalHostName()); + getLogger().error("Invalid domain {}", ip.getCanonicalHostName()); } }); } catch (IOException e) { From 3ce1633c80274c6c05cec08ac9a611c996946464 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 23 Mar 2021 13:05:43 +0530 Subject: [PATCH 04/18] make failover work --- .../SCMBlockLocationFailoverProxyProvider.java | 11 ++++++++++- ...MContainerLocationFailoverProxyProvider.java | 11 ++++++++++- .../org/apache/hadoop/hdds/utils/HAUtils.java | 17 ++++++++++++++++- .../hdds/scm/cli/ContainerOperationClient.java | 10 ++-------- .../hadoop/ozone/MiniOzoneClusterImpl.java | 3 ++- .../apache/hadoop/ozone/om/OzoneManager.java | 9 +-------- .../hadoop/ozone/freon/BaseFreonGenerator.java | 12 +++--------- 7 files changed, 44 insertions(+), 29 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java index d982cf50d2d6..ff7392281d65 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java @@ -70,11 +70,20 @@ public class SCMBlockLocationFailoverProxyProvider implements private final int maxRetryCount; private final long retryInterval; + private final UserGroupInformation ugi; + public SCMBlockLocationFailoverProxyProvider(ConfigurationSource conf) { this.conf = conf; this.scmVersion = RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class); + try { + this.ugi = UserGroupInformation.getCurrentUser(); + } catch (IOException ex) { + LOG.error("Unable to fetch user credentials from UGI", ex); + throw new RuntimeException(ex); + } + // Set some constant for non-HA. if (scmServiceId == null) { scmServiceId = SCM_DUMMY_SERVICE_ID; @@ -231,7 +240,7 @@ private ScmBlockLocationProtocolPB createSCMProxy( RPC.setProtocolEngine(hadoopConf, ScmBlockLocationProtocolPB.class, ProtobufRpcEngine.class); return RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion, - scmAddress, UserGroupInformation.getCurrentUser(), hadoopConf, + scmAddress, ugi, hadoopConf, NetUtils.getDefaultSocketFactory(hadoopConf), (int)conf.getObject(SCMClientConfig.class).getRpcTimeOut()); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java index ff1312257e46..05b0d895df7d 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java @@ -70,9 +70,18 @@ public class SCMContainerLocationFailoverProxyProvider implements private final int maxRetryCount; private final long retryInterval; + private final UserGroupInformation ugi; + public SCMContainerLocationFailoverProxyProvider(ConfigurationSource conf) { this.conf = conf; + + try { + this.ugi = UserGroupInformation.getCurrentUser(); + } catch (IOException ex) { + LOG.error("Unable to fetch user credentials from UGI", ex); + throw new RuntimeException(ex); + } this.scmVersion = RPC.getProtocolVersion( StorageContainerLocationProtocolPB.class); @@ -237,7 +246,7 @@ private StorageContainerLocationProtocolPB createSCMProxy( ProtobufRpcEngine.class); return RPC.getProxy( StorageContainerLocationProtocolPB.class, - scmVersion, scmAddress, UserGroupInformation.getCurrentUser(), + scmVersion, scmAddress, ugi, hadoopConf, NetUtils.getDefaultSocketFactory(hadoopConf), (int)scmClientConfig.getRpcTimeOut()); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java index bf1578731626..0ad2a6eee3ae 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java @@ -29,8 +29,11 @@ import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; +import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.proxy.SCMBlockLocationFailoverProxyProvider; +import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; @@ -42,6 +45,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneSecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.FileUtils; @@ -112,7 +116,7 @@ public static boolean addSCM(OzoneConfiguration conf, AddSCMRequest request, * @throws IOException */ public static ScmBlockLocationProtocol getScmBlockClient( - OzoneConfiguration conf) throws IOException { + OzoneConfiguration conf) { ScmBlockLocationProtocolClientSideTranslatorPB scmBlockLocationClient = new ScmBlockLocationProtocolClientSideTranslatorPB( new SCMBlockLocationFailoverProxyProvider(conf)); @@ -121,6 +125,17 @@ public static ScmBlockLocationProtocol getScmBlockClient( conf); } + public static StorageContainerLocationProtocol getScmContainerClient( + ConfigurationSource conf) { + SCMContainerLocationFailoverProxyProvider proxyProvider = + new SCMContainerLocationFailoverProxyProvider(conf); + StorageContainerLocationProtocol scmContainerClient = + TracingUtil.createProxy( + new StorageContainerLocationProtocolClientSideTranslatorPB( + proxyProvider), StorageContainerLocationProtocol.class, conf); + return scmContainerClient; + } + /** * Replace the current DB with the new DB checkpoint. * diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index 7555d105e4fa..725b7fe7d712 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ozone.OzoneSecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,14 +113,7 @@ private XceiverClientManager newXCeiverClientManager(ConfigurationSource conf) public static StorageContainerLocationProtocol newContainerRpcClient( ConfigurationSource configSource) { - SCMContainerLocationFailoverProxyProvider proxyProvider = - new SCMContainerLocationFailoverProxyProvider(configSource); - - StorageContainerLocationProtocolClientSideTranslatorPB client = - new StorageContainerLocationProtocolClientSideTranslatorPB( - proxyProvider); - return TracingUtil.createProxy( - client, StorageContainerLocationProtocol.class, configSource); + return HAUtils.getScmContainerClient(configSource); } @Override diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index b0ce55d2afb3..af132d697eaf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -66,6 +66,7 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.recon.ConfigurationProvider; import org.apache.hadoop.ozone.recon.ReconServer; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.test.GenericTestUtils; @@ -329,7 +330,7 @@ public OzoneClient getRpcClient() throws IOException { */ @Override public StorageContainerLocationProtocolClientSideTranslatorPB - getStorageContainerLocationClient() { + getStorageContainerLocationClient() throws IOException { InetSocketAddress address = scm.getClientRpcAddress(); LOG.info( "Creating StorageContainerLocationProtocol RPC client with address {}", diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index cd2959e45211..1eb23af26fee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -846,17 +846,10 @@ private static ScmBlockLocationProtocol getScmBlockClient( * Returns a scm container client. * * @return {@link StorageContainerLocationProtocol} - * @throws IOException */ private static StorageContainerLocationProtocol getScmContainerClient( OzoneConfiguration conf) { - SCMContainerLocationFailoverProxyProvider proxyProvider = - new SCMContainerLocationFailoverProxyProvider(conf); - StorageContainerLocationProtocol scmContainerClient = - TracingUtil.createProxy( - new StorageContainerLocationProtocolClientSideTranslatorPB( - proxyProvider), StorageContainerLocationProtocol.class, conf); - return scmContainerClient; + return HAUtils.getScmContainerClient(conf); } /** diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java index 3e8814841d07..dbbe670d288e 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; import org.apache.hadoop.hdds.tracing.TracingUtil; +import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ozone.client.OzoneClient; @@ -342,15 +343,8 @@ public OzoneManagerProtocolClientSideTranslatorPB createOmClient( } public StorageContainerLocationProtocol createStorageContainerLocationClient( - OzoneConfiguration ozoneConf) { - SCMContainerLocationFailoverProxyProvider proxyProvider = - new SCMContainerLocationFailoverProxyProvider(ozoneConf); - StorageContainerLocationProtocol client = - TracingUtil.createProxy( - new StorageContainerLocationProtocolClientSideTranslatorPB( - proxyProvider), - StorageContainerLocationProtocol.class, ozoneConf); - return client; + OzoneConfiguration ozoneConf) throws IOException { + return HAUtils.getScmContainerClient(ozoneConf); } public static Pipeline findPipelineForTest(String pipelineId, From 7c4a23c127d17429e420d30ee46ae18fb2a891db Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 23 Mar 2021 13:06:43 +0530 Subject: [PATCH 05/18] rename to ozonesecure-ha --- .../src/main/compose/{ozonesecure-om-ha => ozonesecure-ha}/.env | 0 .../{ozonesecure-om-ha => ozonesecure-ha}/docker-compose.yaml | 0 .../compose/{ozonesecure-om-ha => ozonesecure-ha}/docker-config | 0 .../main/compose/{ozonesecure-om-ha => ozonesecure-ha}/test.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename hadoop-ozone/dist/src/main/compose/{ozonesecure-om-ha => ozonesecure-ha}/.env (100%) rename hadoop-ozone/dist/src/main/compose/{ozonesecure-om-ha => ozonesecure-ha}/docker-compose.yaml (100%) rename hadoop-ozone/dist/src/main/compose/{ozonesecure-om-ha => ozonesecure-ha}/docker-config (100%) rename hadoop-ozone/dist/src/main/compose/{ozonesecure-om-ha => ozonesecure-ha}/test.sh (100%) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/.env rename to hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-compose.yaml rename to hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config rename to hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh similarity index 100% rename from hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/test.sh rename to hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test.sh From 1ef8de3d163aceaf96f4d034908bab63266a1048 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 23 Mar 2021 13:09:12 +0530 Subject: [PATCH 06/18] fix cs --- .../src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java | 1 - .../apache/hadoop/hdds/scm/cli/ContainerOperationClient.java | 4 ---- .../java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java | 1 - .../main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 3 --- .../org/apache/hadoop/ozone/freon/BaseFreonGenerator.java | 3 --- 5 files changed, 12 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java index 0ad2a6eee3ae..908143eedef7 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.ozone.OzoneSecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.FileUtils; diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index 725b7fe7d712..d4d54a0165a4 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -38,14 +38,10 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; -import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ozone.OzoneSecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index af132d697eaf..ec51c42667d0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -66,7 +66,6 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.recon.ConfigurationProvider; import org.apache.hadoop.ozone.recon.ReconServer; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.test.GenericTestUtils; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 1eb23af26fee..382d4e7ed88c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -69,8 +69,6 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; -import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.client.OMCertificateClient; @@ -78,7 +76,6 @@ import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl; import org.apache.hadoop.hdds.server.http.RatisDropwizardExports; -import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java index dbbe670d288e..6d249687fa06 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java @@ -32,9 +32,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; -import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; -import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; From 21f1d7c873263ed70134a366301a19289aef9e0c Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Wed, 24 Mar 2021 15:59:01 +0530 Subject: [PATCH 07/18] Fix bug in failover proxyprovider of scmsecurity and donot retry on connection errors --- .../proxy/SCMBlockLocationFailoverProxyProvider.java | 11 +++++++++-- .../SCMContainerLocationFailoverProxyProvider.java | 10 ++++++++-- .../SCMSecurityProtocolFailoverProxyProvider.java | 2 +- .../hdds/scm/server/SCMSecurityProtocolServer.java | 5 +++-- 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java index ff7392281d65..bdd7a200ca91 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMBlockLocationFailoverProxyProvider.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB; import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource; import org.apache.hadoop.io.retry.FailoverProxyProvider; +import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryPolicy.RetryAction; import org.apache.hadoop.ipc.ProtobufRpcEngine; @@ -239,10 +240,16 @@ private ScmBlockLocationProtocolPB createSCMProxy( LegacyHadoopConfigurationSource.asHadoopConfiguration(conf); RPC.setProtocolEngine(hadoopConf, ScmBlockLocationProtocolPB.class, ProtobufRpcEngine.class); - return RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion, + // FailoverOnNetworkException ensures that the IPC layer does not attempt + // retries on the same OM in case of connection exception. This retry + // policy essentially results in TRY_ONCE_THEN_FAIL. + RetryPolicy connectionRetryPolicy = RetryPolicies + .failoverOnNetworkException(0); + return RPC.getProtocolProxy(ScmBlockLocationProtocolPB.class, scmVersion, scmAddress, ugi, hadoopConf, NetUtils.getDefaultSocketFactory(hadoopConf), - (int)conf.getObject(SCMClientConfig.class).getRpcTimeOut()); + (int)conf.getObject(SCMClientConfig.class).getRpcTimeOut(), + connectionRetryPolicy).getProxy(); } public RetryPolicy getSCMBlockLocationRetryPolicy(String newLeader) { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java index 05b0d895df7d..96f86b819f1b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMContainerLocationFailoverProxyProvider.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB; import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource; import org.apache.hadoop.io.retry.FailoverProxyProvider; +import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; @@ -244,11 +245,16 @@ private StorageContainerLocationProtocolPB createSCMProxy( LegacyHadoopConfigurationSource.asHadoopConfiguration(conf); RPC.setProtocolEngine(hadoopConf, StorageContainerLocationProtocolPB.class, ProtobufRpcEngine.class); - return RPC.getProxy( + // FailoverOnNetworkException ensures that the IPC layer does not attempt + // retries on the same OM in case of connection exception. This retry + // policy essentially results in TRY_ONCE_THEN_FAIL. + RetryPolicy connectionRetryPolicy = RetryPolicies + .failoverOnNetworkException(0); + return RPC.getProtocolProxy( StorageContainerLocationProtocolPB.class, scmVersion, scmAddress, ugi, hadoopConf, NetUtils.getDefaultSocketFactory(hadoopConf), - (int)scmClientConfig.getRpcTimeOut()); + (int)scmClientConfig.getRpcTimeOut(), connectionRetryPolicy).getProxy(); } public RetryPolicy getRetryPolicy() { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMSecurityProtocolFailoverProxyProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMSecurityProtocolFailoverProxyProvider.java index a2d2fb35e4a4..91e0dca87d18 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMSecurityProtocolFailoverProxyProvider.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/proxy/SCMSecurityProtocolFailoverProxyProvider.java @@ -199,7 +199,7 @@ public void performFailoverToNextProxy() { * @return the new proxy index */ private synchronized int incrementProxyIndex() { - currentProxyIndex = (currentProxyIndex + 1) % scmProxies.size(); + currentProxyIndex = (currentProxyIndex + 1) % scmProxyInfoMap.size(); currentProxySCMNodeId = scmNodeIds.get(currentProxyIndex); return currentProxyIndex; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java index 8ca5cebb704e..d08fe4bef2fe 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java @@ -231,7 +231,8 @@ public String getCertificate(String certSerialId) throws IOException { certSerialId); try { X509Certificate certificate = - rootCertificateServer.getCertificate(certSerialId); + storageContainerManager.getScmCertificateServer() + .getCertificate(certSerialId); if (certificate != null) { return CertificateCodec.getPEMEncodedString(certificate); } @@ -239,7 +240,7 @@ public String getCertificate(String certSerialId) throws IOException { throw new SCMSecurityException("getCertificate operation failed. ", e, GET_CERTIFICATE_FAILED); } - LOGGER.debug("Certificate with serial id {} not found.", certSerialId); + LOGGER.info("Certificate with serial id {} not found.", certSerialId); throw new SCMSecurityException("Certificate not found", CERTIFICATE_NOT_FOUND); } From 7c57bf9e3f241f2759df834e9874b15a5ea9e084 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Wed, 24 Mar 2021 16:20:22 +0530 Subject: [PATCH 08/18] fix review comments --- .../hadoop/hdds/scm/XceiverClientGrpc.java | 4 ++-- .../hadoop/hdds/scm/XceiverClientRatis.java | 4 ++-- .../apache/hadoop/hdds/ratis/RatisHelper.java | 4 ++-- .../apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 4 ++-- .../apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 16 ---------------- .../container/ozoneimpl/OzoneContainer.java | 4 +--- .../client/DefaultCertificateClient.java | 2 +- 7 files changed, 10 insertions(+), 28 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java index c3ec5d1de50c..4ff47e0bf460 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java @@ -96,7 +96,7 @@ public class XceiverClientGrpc extends XceiverClientSpi { * @param caCert - SCM ca certificate. */ public XceiverClientGrpc(Pipeline pipeline, ConfigurationSource config, - List caCert) { + List caCerts) { super(); Preconditions.checkNotNull(pipeline); Preconditions.checkNotNull(config); @@ -114,7 +114,7 @@ public XceiverClientGrpc(Pipeline pipeline, ConfigurationSource config, this.topologyAwareRead = config.getBoolean( OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY, OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_DEFAULT); - this.caCerts = caCert; + this.caCerts = caCerts; this.getBlockDNcache = new ConcurrentHashMap<>(); } diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java index cece6a6fac50..6982d41fbce5 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java @@ -81,13 +81,13 @@ public static XceiverClientRatis newXceiverClientRatis( public static XceiverClientRatis newXceiverClientRatis( org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline, - ConfigurationSource ozoneConf, List caCert) { + ConfigurationSource ozoneConf, List caCerts) { final String rpcType = ozoneConf .get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY, ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT); final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf); final GrpcTlsConfig tlsConfig = RatisHelper.createTlsClientConfig(new - SecurityConfig(ozoneConf), caCert); + SecurityConfig(ozoneConf), caCerts); return new XceiverClientRatis(pipeline, SupportedRpcType.valueOfIgnoreCase(rpcType), retryPolicy, tlsConfig, ozoneConf); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java index 4b648a102330..e310cc9f7ddd 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java @@ -294,11 +294,11 @@ private static Map getDatanodeRatisPrefixProps( // For External gRPC client to server with gRPC TLS. // No mTLS for external client as SCM CA does not issued certificates for them public static GrpcTlsConfig createTlsClientConfig(SecurityConfig conf, - List caCert) { + List caCerts) { GrpcTlsConfig tlsConfig = null; if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { tlsConfig = new GrpcTlsConfig(null, null, - caCert, false); + caCerts, false); } return tlsConfig; } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java index 915e234e9353..594985e9a662 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java @@ -181,7 +181,7 @@ public static OzoneConfiguration removeSelfId( */ public static Collection getSCMNodeIds( ConfigurationSource configuration) { - String scmServiceId = SCMHAUtils.getScmServiceId(configuration); - return SCMHAUtils.getSCMNodeIds(configuration, scmServiceId); + String scmServiceId = getScmServiceId(configuration); + return getSCMNodeIds(configuration, scmServiceId); } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java index 5dacf9ed8ae2..e71d05e2cb55 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java @@ -247,20 +247,4 @@ public String getScmSecurityAddress() { public String getScmDatanodeAddress() { return scmDatanodeAddress; } - - public static SCMNodeInfo getScmNodeInfo(ConfigurationSource conf, - String nodeId) { - List scmNodeInfoList = SCMNodeInfo.buildNodeInfo(conf); - Preconditions.checkState(scmNodeInfoList.size() >=1); - if (SCMHAUtils.getScmServiceId(conf) != null) { - for (SCMNodeInfo scmNodeInfo : scmNodeInfoList) { - if (scmNodeInfo.getNodeId().equals(nodeId)) { - return scmNodeInfo; - } - } - } else { - return scmNodeInfoList.get(0); - } - return null; - } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 91ed37364743..87551ff45cb3 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -172,7 +172,7 @@ public OzoneContainer( new BlockDeletingService(this, svcInterval.toMillis(), serviceTimeout, TimeUnit.MILLISECONDS, config); - List< X509Certificate > x509Certificates; + List< X509Certificate > x509Certificates = null; if (certClient != null) { List pemEncodedCerts = HAUtils.buildCAList(certClient, conf); x509Certificates = new ArrayList<>(pemEncodedCerts.size()); @@ -184,8 +184,6 @@ public OzoneContainer( throw new IOException(ex); } } - } else { - x509Certificates = null; } tlsClientConfig = RatisHelper.createTlsClientConfig(secConf, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index 07a5d05e4ea0..f9de60575abb 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -868,7 +868,7 @@ public void storeRootCACertificate(String pemEncodedCert, boolean force) } @Override - public List listCA() throws IOException { + public synchronized List listCA() throws IOException { if (pemEncodedCACerts == null) { updateCAList(); } From ec43af1223a7a980ac1a31cac8c0dc1da2740f22 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Wed, 24 Mar 2021 16:34:23 +0530 Subject: [PATCH 09/18] fix review comments --- .../hadoop/hdds/scm/ha/SCMNodeInfo.java | 1 - .../client/DefaultCertificateClient.java | 20 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java index e71d05e2cb55..f3191ea41327 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdds.scm.ha; -import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.ozone.ha.ConfUtils; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index f9de60575abb..00244eeb5f59 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -41,6 +41,8 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; @@ -93,6 +95,7 @@ public abstract class DefaultCertificateClient implements CertificateClient { private String rootCaCertId; private String component; private List pemEncodedCACerts = null; + private final Lock lock; DefaultCertificateClient(SecurityConfig securityConfig, Logger log, String certSerialId, String component) { @@ -103,6 +106,7 @@ public abstract class DefaultCertificateClient implements CertificateClient { this.certificateMap = new ConcurrentHashMap<>(); this.certSerialId = certSerialId; this.component = component; + lock = new ReentrantLock(); loadAllCertificates(); } @@ -868,15 +872,21 @@ public void storeRootCACertificate(String pemEncodedCert, boolean force) } @Override - public synchronized List listCA() throws IOException { - if (pemEncodedCACerts == null) { - updateCAList(); + public List listCA() throws IOException { + lock.lock(); + try { + if (pemEncodedCACerts == null) { + updateCAList(); + } + return pemEncodedCACerts; + }finally { + lock.unlock(); } - return pemEncodedCACerts; } @Override public List updateCAList() throws IOException { + lock.lock(); try { SCMSecurityProtocol scmSecurityProtocolClient = HddsServerUtil.getScmSecurityClient( @@ -888,6 +898,8 @@ public List updateCAList() throws IOException { getLogger().error("Error during updating CA list", e); throw new CertificateException("Error during updating CA list", e, CERTIFICATE_ERROR); + } finally { + lock.unlock(); } } } From 2f101a0f3122c90df82f0c0b859f56b49b282aa3 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Wed, 24 Mar 2021 17:04:17 +0530 Subject: [PATCH 10/18] fix review comment on scm cert --- .../exception/SCMSecurityException.java | 3 +- .../proto/ScmServerSecurityProtocol.proto | 1 + ...ecurityProtocolServerSideTranslatorPB.java | 2 ++ .../scm/server/SCMSecurityProtocolServer.java | 30 ++++++++++++------- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java index d75acc4abb1b..a8e5cb491272 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java @@ -108,6 +108,7 @@ public enum ErrorCode { DEFAULT, MISSING_BLOCK_TOKEN, BLOCK_TOKEN_VERIFICATION_FAILED, - GET_ROOT_CA_CERT_FAILED + GET_ROOT_CA_CERT_FAILED, + PRIMARY_SCM_IS_NOT_LEADER } } diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto index 31aac909dbc9..fe50795c4cdc 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto @@ -100,6 +100,7 @@ enum Status { MISSING_BLOCK_TOKEN = 13; BLOCK_TOKEN_VERIFICATION_FAILED = 14; GET_ROOT_CA_CERTIFICATE_FAILED = 15; + PRIMARY_SCM_IS_NOT_LEADER = 16; } /** * This message is send by data node to prove its identity and get an SCM diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java index 8a71eb5c2b8e..a6e41433a989 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/SCMSecurityProtocolServerSideTranslatorPB.java @@ -44,6 +44,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.Type.GetSCMCertificate; + /** * This class is the server-side translator that forwards requests received on * {@link SCMSecurityProtocolPB} to the {@link diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java index d08fe4bef2fe..f27619797f62 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java @@ -57,6 +57,7 @@ import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.CERTIFICATE_NOT_FOUND; import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CA_CERT_FAILED; import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CERTIFICATE_FAILED; +import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.PRIMARY_SCM_IS_NOT_LEADER; import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateApprover.ApprovalType.KERBEROS_TRUSTED; /** @@ -154,18 +155,27 @@ public String getOMCertificate(OzoneManagerDetailsProto omDetails, public String getSCMCertificate(ScmNodeDetailsProto scmNodeDetails, String certSignReq) throws IOException { Objects.requireNonNull(scmNodeDetails); - LOGGER.info("Processing CSR for scm {}, nodeId: {}", - scmNodeDetails.getHostName(), scmNodeDetails.getScmNodeId()); + String primaryScmId = + storageContainerManager.getScmStorageConfig().getPrimaryScmNodeId(); - // Check clusterID - if (!storageContainerManager.getClusterId().equals( - scmNodeDetails.getClusterId())) { - throw new IOException("SCM ClusterId mismatch. Peer SCM ClusterId " + - scmNodeDetails.getClusterId() + ", primary SCM ClusterId " - + storageContainerManager.getClusterId()); - } + if (primaryScmId != null && + primaryScmId.equals(storageContainerManager.getScmId())) { + LOGGER.info("Processing CSR for scm {}, nodeId: {}", + scmNodeDetails.getHostName(), scmNodeDetails.getScmNodeId()); + + // Check clusterID + if (!storageContainerManager.getClusterId().equals( + scmNodeDetails.getClusterId())) { + throw new IOException("SCM ClusterId mismatch. Peer SCM ClusterId " + + scmNodeDetails.getClusterId() + ", primary SCM ClusterId " + + storageContainerManager.getClusterId()); + } - return getEncodedCertToString(certSignReq, NodeType.SCM); + return getEncodedCertToString(certSignReq, NodeType.SCM); + } else { + throw new SCMSecurityException("Get SCM Certificate can be run only " + + "primary SCM", PRIMARY_SCM_IS_NOT_LEADER); + } } From 726f13d9251c3d864762003a6edee302edb9cb3f Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 29 Mar 2021 11:16:31 +0530 Subject: [PATCH 11/18] fix review comments --- .../hadoop/hdds/scm/XceiverClientManager.java | 18 ++----- .../exception/SCMSecurityException.java | 2 +- .../hadoop/ozone/OzoneSecurityUtil.java | 19 ++++++++ .../server/ratis/XceiverServerRatis.java | 4 +- .../container/ozoneimpl/OzoneContainer.java | 13 +---- .../authority/DefaultApprover.java | 2 +- .../org/apache/hadoop/hdds/utils/HAUtils.java | 39 ++++++++++----- .../proto/ScmServerSecurityProtocol.proto | 2 +- .../hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +- .../hdds/scm/ha/SCMRatisServerImpl.java | 2 +- .../scm/server/SCMSecurityProtocolServer.java | 24 +++++----- .../hdds/scm/server/SCMStorageConfig.java | 4 ++ .../scm/server/StorageContainerManager.java | 48 +++---------------- .../server/TestSCMSecurityProtocolServer.java | 3 +- .../scm/cli/ContainerOperationClient.java | 4 +- .../apache/hadoop/hdds/scm/cli/ScmOption.java | 1 - .../hadoop/ozone/client/rpc/RpcClient.java | 4 +- .../om/ratis/OzoneManagerRatisServer.java | 5 +- 18 files changed, 92 insertions(+), 104 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java index 341ab6f4f102..f0169723b105 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java @@ -20,9 +20,7 @@ import java.io.Closeable; import java.io.IOException; -import java.security.cert.CertificateException; import java.security.cert.X509Certificate; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; @@ -33,8 +31,6 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.hdds.security.exception.SCMSecurityException; -import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -88,23 +84,15 @@ public XceiverClientManager(ConfigurationSource conf) throws IOException { public XceiverClientManager(ConfigurationSource conf, ScmClientConfig clientConf, - List caCertPems) throws IOException { + List caCerts) throws IOException { Preconditions.checkNotNull(clientConf); Preconditions.checkNotNull(conf); long staleThresholdMs = clientConf.getStaleThreshold(MILLISECONDS); this.conf = conf; this.isSecurityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf); if (isSecurityEnabled) { - Preconditions.checkNotNull(caCertPems); - try { - this.caCerts = new ArrayList<>(caCertPems.size()); - for (String cert : caCertPems) { - this.caCerts.add(CertificateCodec.getX509Cert(cert)); - } - } catch (CertificateException ex) { - throw new SCMSecurityException("Error: Fail to get SCM CA certificate", - ex); - } + Preconditions.checkNotNull(caCerts); + this.caCerts = caCerts; } this.clientCache = CacheBuilder.newBuilder() diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java index a8e5cb491272..7e008afc416b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java @@ -109,6 +109,6 @@ public enum ErrorCode { MISSING_BLOCK_TOKEN, BLOCK_TOKEN_VERIFICATION_FAILED, GET_ROOT_CA_CERT_FAILED, - PRIMARY_SCM_IS_NOT_LEADER + NOT_A_PRIMARY_SCM } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java index 726862cea730..10fbd150cae8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java @@ -23,6 +23,8 @@ import java.net.InetAddress; import java.net.NetworkInterface; import java.nio.file.Path; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; import java.util.Enumeration; @@ -39,6 +41,8 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_SECURITY_ENABLED_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; + +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -120,4 +124,19 @@ public static List getValidInetsForCurrentHost() return hostIps; } + + public static List convertToX509( + List pemEncodedCerts) throws IOException { + List x509Certificates = + new ArrayList<>(pemEncodedCerts.size()); + for (String cert : pemEncodedCerts) { + try { + x509Certificates.add(CertificateCodec.getX509Certificate(cert)); + } catch (CertificateException ex) { + LOG.error("Error while converting to X509 format", ex); + throw new IOException(ex); + } + } + return x509Certificates; + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 84f3cae8d767..8f779a59e11e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -466,7 +466,7 @@ private static Parameters createTlsParameters(SecurityConfig conf, Parameters parameters = new Parameters(); if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { - ArrayList< X509Certificate > listCA = new ArrayList<>(); + List listCA = new ArrayList<>(); listCA.add(caClient.getCACertificate()); if (caClient.getRootCACertificate() != null) { listCA.add(caClient.getRootCACertificate()); @@ -479,7 +479,7 @@ private static Parameters createTlsParameters(SecurityConfig conf, GrpcTlsConfig clientConfig = new GrpcTlsConfig( caClient.getPrivateKey(), caClient.getCertificate(), - caClient.getCACertificate(), false); + listCA, false); GrpcConfigKeys.Client.setTlsConf(parameters, clientConfig); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 87551ff45cb3..5bb9fb16d9ce 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.container.ozoneimpl; import java.io.IOException; -import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.time.Duration; import java.util.ArrayList; @@ -42,7 +41,6 @@ import org.apache.hadoop.hdds.security.token.BlockTokenVerifier; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; -import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; @@ -174,16 +172,7 @@ public OzoneContainer( List< X509Certificate > x509Certificates = null; if (certClient != null) { - List pemEncodedCerts = HAUtils.buildCAList(certClient, conf); - x509Certificates = new ArrayList<>(pemEncodedCerts.size()); - for (String cert : pemEncodedCerts) { - try { - x509Certificates.add(CertificateCodec.getX509Certificate(cert)); - } catch (CertificateException ex) { - LOG.error("Error while fetching CA", ex); - throw new IOException(ex); - } - } + x509Certificates = HAUtils.buildCAX509List(certClient, conf); } tlsClientConfig = RatisHelper.createTlsClientConfig(secConf, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultApprover.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultApprover.java index 3ab0911c2a8c..35122c7b6f26 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultApprover.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultApprover.java @@ -77,7 +77,7 @@ public DefaultApprover(PKIProfile pkiProfile, SecurityConfig config) { * @param scmId - SCM id. * @param clusterId - Cluster id. * @return Signed Certificate. - * @throws IOException - On Error + * @throws IOException - On Error * @throws OperatorCreationException - on Error. */ @SuppressWarnings("ParameterNumber") diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java index 908143eedef7..212250f936a9 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java @@ -58,6 +58,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -446,17 +447,6 @@ public static List buildCAList(CertificateClient certClient, return caCertPemList; } - /** - * Build CA list which need to be passed to client. - * - * @param configuration - * @return list of CA - * @throws IOException - */ - public static List buildCAList(ConfigurationSource configuration) - throws IOException { - return buildCAList(null, configuration); - } private static void checkCertCount(int certCount, int expectedCount) throws SCMSecurityException{ @@ -468,5 +458,32 @@ private static void checkCertCount(int certCount, int expectedCount) } } + /** + * Build CA List in the format of X509Certificate. + * If certificate client is null, obtain the list of CA using SCM + * security client, else it uses certificate client. + * @param certClient + * @param conf + * @return list of CA X509Certificates. + * @throws IOException + */ + public static List buildCAX509List( + CertificateClient certClient, + ConfigurationSource conf) throws IOException { + if (certClient != null) { + // Do this here to avoid extra conversion of X509 to pem and again to + // X509 by buildCAList. + if (!SCMHAUtils.isSCMHAEnabled(conf)) { + List x509Certificates = new ArrayList<>(); + if (certClient.getRootCACertificate() != null) { + x509Certificates.add(certClient.getRootCACertificate()); + } + x509Certificates.add(certClient.getCACertificate()); + return x509Certificates; + } + } + List pemEncodedCerts = HAUtils.buildCAList(certClient, conf); + return OzoneSecurityUtil.convertToX509(pemEncodedCerts); + } } diff --git a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto index fe50795c4cdc..e270b27c7d90 100644 --- a/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto +++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerSecurityProtocol.proto @@ -100,7 +100,7 @@ enum Status { MISSING_BLOCK_TOKEN = 13; BLOCK_TOKEN_VERIFICATION_FAILED = 14; GET_ROOT_CA_CERTIFICATE_FAILED = 15; - PRIMARY_SCM_IS_NOT_LEADER = 16; + NOT_A_PRIMARY_SCM = 16; } /** * This message is send by data node to prove its identity and get an SCM diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java index a673808e77b2..e32212527b85 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/HASecurityUtils.java @@ -290,7 +290,7 @@ private static void persistSubCACertificate(OzoneConfiguration config, * @param caClient * @return */ - public static Parameters createServerTlsParameters(SecurityConfig conf, + public static Parameters createSCMServerTlsParameters(SecurityConfig conf, CertificateClient caClient) { Parameters parameters = new Parameters(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java index c30ba7059e25..54a1e0fe6d82 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMRatisServerImpl.java @@ -85,7 +85,7 @@ public class SCMRatisServerImpl implements SCMRatisServer { // trigger leader election. Parameters parameters = - HASecurityUtils.createServerTlsParameters(new SecurityConfig(conf), + HASecurityUtils.createSCMServerTlsParameters(new SecurityConfig(conf), scm.getScmCertificateClient()); this.server = newRaftServer(scm.getScmId(), conf) .setStateMachine(stateMachine) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java index f27619797f62..5c8642d0b824 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java @@ -57,7 +57,7 @@ import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.CERTIFICATE_NOT_FOUND; import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CA_CERT_FAILED; import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.GET_CERTIFICATE_FAILED; -import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.PRIMARY_SCM_IS_NOT_LEADER; +import static org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.NOT_A_PRIMARY_SCM; import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateApprover.ApprovalType.KERBEROS_TRUSTED; /** @@ -71,16 +71,19 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol { private static final Logger LOGGER = LoggerFactory .getLogger(SCMSecurityProtocolServer.class); private final CertificateServer rootCertificateServer; + private final CertificateServer scmCertificateServer; private final RPC.Server rpcServer; private final InetSocketAddress rpcAddress; private final ProtocolMessageMetrics metrics; private final StorageContainerManager storageContainerManager; SCMSecurityProtocolServer(OzoneConfiguration conf, - CertificateServer certificateServer, StorageContainerManager scm) + CertificateServer rootCertificateServer, + CertificateServer scmCertificateServer, StorageContainerManager scm) throws IOException { this.storageContainerManager = scm; - this.rootCertificateServer = certificateServer; + this.rootCertificateServer = rootCertificateServer; + this.scmCertificateServer = scmCertificateServer; final int handlerCount = conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY, ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT); @@ -174,7 +177,7 @@ public String getSCMCertificate(ScmNodeDetailsProto scmNodeDetails, return getEncodedCertToString(certSignReq, NodeType.SCM); } else { throw new SCMSecurityException("Get SCM Certificate can be run only " + - "primary SCM", PRIMARY_SCM_IS_NOT_LEADER); + "primary SCM", NOT_A_PRIMARY_SCM); } } @@ -194,8 +197,8 @@ private String getEncodedCertToString(String certSignReq, NodeType nodeType) future = rootCertificateServer.requestCertificate(certSignReq, KERBEROS_TRUSTED, nodeType); } else { - future = storageContainerManager.getScmCertificateServer() - .requestCertificate(certSignReq, KERBEROS_TRUSTED, nodeType); + future = scmCertificateServer.requestCertificate(certSignReq, + KERBEROS_TRUSTED, nodeType); } try { return CertificateCodec.getPEMEncodedString(future.get()); @@ -241,8 +244,7 @@ public String getCertificate(String certSerialId) throws IOException { certSerialId); try { X509Certificate certificate = - storageContainerManager.getScmCertificateServer() - .getCertificate(certSerialId); + scmCertificateServer.getCertificate(certSerialId); if (certificate != null) { return CertificateCodec.getPEMEncodedString(certificate); } @@ -265,7 +267,7 @@ public String getCACertificate() throws IOException { LOGGER.debug("Getting CA certificate."); try { return CertificateCodec.getPEMEncodedString( - storageContainerManager.getScmCertificateServer().getCACertificate()); + scmCertificateServer.getCACertificate()); } catch (CertificateException e) { throw new SCMSecurityException("getRootCertificate operation failed. ", e, GET_CA_CERT_FAILED); @@ -285,8 +287,8 @@ public String getCACertificate() throws IOException { public List listCertificate(NodeType role, long startSerialId, int count, boolean isRevoked) throws IOException { List certificates = - storageContainerManager.getScmCertificateServer() - .listCertificate(role, startSerialId, count, isRevoked); + scmCertificateServer.listCertificate(role, startSerialId, count, + isRevoked); List results = new ArrayList<>(certificates.size()); for (X509Certificate cert : certificates) { try { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java index 72d252503d8c..4a346ddf6b82 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java @@ -113,4 +113,8 @@ public void setPrimaryScmNodeId(String scmId) throws IOException { public String getPrimaryScmNodeId() { return getStorageInfo().getProperty(PRIMARY_SCM_NODE_ID); } + + public boolean checkPrimarySCMIdInitialized() { + return getPrimaryScmNodeId() != null ? true : false; + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 9c619cba779b..e3931c179281 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -145,8 +145,6 @@ import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD; import static org.apache.hadoop.ozone.OzoneConsts.CRL_SEQUENCE_ID_KEY; -import static org.apache.hadoop.ozone.OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME; -import static org.apache.hadoop.ozone.OzoneConsts.SCM_ROOT_CA_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.SCM_SUB_CA_PREFIX; /** @@ -414,7 +412,7 @@ private StorageContainerManager(OzoneConfiguration conf, } private void initializeCertificateClient() { - if (primaryScmNodeId != null) { + if (scmStorageConfig.checkPrimarySCMIdInitialized()) { scmCertificateClient = new SCMCertificateClient( new SecurityConfig(configuration), scmStorageConfig.getScmCertSerialId()); @@ -597,7 +595,7 @@ private void initializeCAnSecurityProtocol(OzoneConfiguration conf, // If primary SCM node Id is set it means this is a cluster which has // performed init with SCM HA version code. - if (primaryScmNodeId != null) { + if (scmStorageConfig.checkPrimarySCMIdInitialized()) { // Start specific instance SCM CA server. String subject = SCM_SUB_CA_PREFIX + InetAddress.getLocalHost().getHostName(); @@ -652,18 +650,14 @@ certificateStore, new DefaultProfile(), // not be run again after upgrade. So for a upgraded cluster where init // has not happened again we will have setup like before where it has // one CA server which is issuing certificates to DN and OM. - String subject = SCM_ROOT_CA_PREFIX + - InetAddress.getLocalHost().getHostName(); - rootCertificateServer = new DefaultCAServer(subject, - getScmStorageConfig().getClusterID(), - getScmStorageConfig().getScmId(), certificateStore, - new DefaultProfile(), - SCM_ROOT_CA_COMPONENT_NAME); + rootCertificateServer = + HASecurityUtils.initializeRootCertificateServer(conf, + certificateStore, scmStorageConfig); scmCertificateServer = rootCertificateServer; } securityProtocolServer = new SCMSecurityProtocolServer(conf, - rootCertificateServer, this); + rootCertificateServer, scmCertificateServer, this); } public CertificateServer getRootCertificateServer() { @@ -730,36 +724,6 @@ private static void loginAsSCMUserIfSecurityEnabled( } } - /** - * This function creates/initializes a certificate server as needed. - * This function is idempotent, so calling this again and again after the - * server is initialized is not a problem. - * - * @param clusterID - Cluster ID - * @param scmID - SCM ID - */ - private CertificateServer initializeCertificateServer(String clusterID, - String scmID) throws IOException { - // TODO: Support Certificate Server loading via Class Name loader. - // So it is easy to use different Certificate Servers if needed. - String subject = SCM_ROOT_CA_PREFIX + - InetAddress.getLocalHost().getHostName(); - if(this.scmMetadataStore == null) { - LOG.error("Cannot initialize Certificate Server without a valid meta " + - "data layer."); - throw new SCMException("Cannot initialize CA without a valid metadata " + - "store", ResultCodes.SCM_NOT_INITIALIZED); - } - - CertificateStore certStore = - new SCMCertStore.Builder().setMetadaStore(scmMetadataStore) - .setRatisServer(scmHAManager.getRatisServer()) - .setCRLSequenceId(getLastSequenceIdForCRL()).build(); - - return new DefaultCAServer(subject, clusterID, scmID, certStore, - new DefaultProfile(), SCM_ROOT_CA_COMPONENT_NAME); - } - long getLastSequenceIdForCRL() throws IOException { Long sequenceId = scmMetadataStore.getCRLSequenceIdTable().get(CRL_SEQUENCE_ID_KEY); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java index 10247563ff89..4fa1d4636b8a 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java @@ -41,7 +41,8 @@ public void setUp() throws Exception { config = new OzoneConfiguration(); config.set(OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY, OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT + ":0"); - securityProtocolServer = new SCMSecurityProtocolServer(config, null, null); + securityProtocolServer = new SCMSecurityProtocolServer(config, null, + null, null); } @After diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java index d4d54a0165a4..b47c45d37dbb 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdds.scm.cli; import java.io.IOException; +import java.security.cert.X509Certificate; import java.util.List; import java.util.Map; @@ -97,7 +98,8 @@ private XceiverClientManager newXCeiverClientManager(ConfigurationSource conf) throws IOException { XceiverClientManager manager; if (OzoneSecurityUtil.isSecurityEnabled(conf)) { - List caCertificates = HAUtils.buildCAList(conf); + List caCertificates = + HAUtils.buildCAX509List(null, conf); manager = new XceiverClientManager(conf, conf.getObject(XceiverClientManager.ScmClientConfig.class), caCertificates); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java index cc84d536d0be..d4396818a0f8 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java @@ -61,7 +61,6 @@ public ScmClient createScmClient() { return new ContainerOperationClient(conf); } catch (IOException ex) { - ex.printStackTrace(); throw new IllegalArgumentException("Can't create SCM client", ex); } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index d3d93a9324bc..1a61f3c06e5f 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -183,8 +183,10 @@ public RpcClient(ConfigurationSource conf, String omServiceId) caCertPems = Collections.singletonList(caCertPem); } + this.xceiverClientManager = new XceiverClientManager(conf, - conf.getObject(XceiverClientManager.ScmClientConfig.class), caCertPems); + conf.getObject(XceiverClientManager.ScmClientConfig.class), + OzoneSecurityUtil.convertToX509(caCertPems)); int configuredChunkSize = (int) conf .getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java index 127e8e001f85..fa510f789b96 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.tracing.TracingUtil; +import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ipc.ProtobufRpcEngine.Server; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; @@ -648,11 +649,11 @@ public RaftGroupId getRaftGroupId() { } private static Parameters createServerTlsParameters(SecurityConfig conf, - CertificateClient caClient) { + CertificateClient caClient) throws IOException { Parameters parameters = new Parameters(); if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { - ArrayList< X509Certificate > listCA = new ArrayList<>(); + List listCA = new ArrayList<>(); listCA.add(caClient.getCACertificate()); if (caClient.getRootCACertificate() != null) { listCA.add(caClient.getRootCACertificate()); From 9b4fe67a4ca4ad59d872133107dc0630adca126e Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 29 Mar 2021 11:29:18 +0530 Subject: [PATCH 12/18] fix cs --- .../apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java index fa510f789b96..5a76e0714a4b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.tracing.TracingUtil; -import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ipc.ProtobufRpcEngine.Server; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; From cccfb5f05a73279d761ddf24d2ec9a5bbcf3c9ae Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 29 Mar 2021 12:25:15 +0530 Subject: [PATCH 13/18] fix code issue causing test failure and init of certificate server in upgrade case --- .../hadoop/hdds/scm/XceiverClientGrpc.java | 2 +- .../scm/server/StorageContainerManager.java | 2 -- .../hadoop/ozone/client/rpc/RpcClient.java | 18 ++++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java index 4ff47e0bf460..e714f7fbb93e 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java @@ -93,7 +93,7 @@ public class XceiverClientGrpc extends XceiverClientSpi { * * @param pipeline - Pipeline that defines the machines. * @param config -- Ozone Config - * @param caCert - SCM ca certificate. + * @param caCerts - SCM ca certificate. */ public XceiverClientGrpc(Pipeline pipeline, ConfigurationSource config, List caCerts) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index e3931c179281..72892e7bc6a9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -618,8 +618,6 @@ certificateStore, new DefaultProfile(), rootCertificateServer = HASecurityUtils.initializeRootCertificateServer( conf, certificateStore, scmStorageConfig); - rootCertificateServer.init(new SecurityConfig(configuration), - CertificateServer.CAType.SELF_SIGNED_CA); } BigInteger certSerial = diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 1a61f3c06e5f..c5d591cd6a62 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -25,6 +25,7 @@ import java.net.URI; import java.security.InvalidKeyException; import java.security.SecureRandom; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -173,20 +174,21 @@ public RpcClient(ConfigurationSource conf, String omServiceId) ); dtService = omTransport.getDelegationTokenService(); ServiceInfoEx serviceInfoEx = ozoneManagerClient.getServiceInfo(); - String caCertPem = null; - List caCertPems = null; + List x509Certificates = null; if (OzoneSecurityUtil.isSecurityEnabled(conf)) { + String caCertPem = null; + List caCertPems = null; caCertPem = serviceInfoEx.getCaCertificate(); + caCertPems = serviceInfoEx.getCaCertPemList(); + if (caCertPems == null || caCertPems.isEmpty()) { + caCertPems = Collections.singletonList(caCertPem); + } + x509Certificates = OzoneSecurityUtil.convertToX509(caCertPems); } - caCertPems = serviceInfoEx.getCaCertPemList(); - if (caCertPems == null || caCertPems.isEmpty()) { - caCertPems = Collections.singletonList(caCertPem); - } - this.xceiverClientManager = new XceiverClientManager(conf, conf.getObject(XceiverClientManager.ScmClientConfig.class), - OzoneSecurityUtil.convertToX509(caCertPems)); + x509Certificates); int configuredChunkSize = (int) conf .getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, From 9cef602c5ee308f82ebf72dd0ba19c2c6d51ae81 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 29 Mar 2021 12:27:04 +0530 Subject: [PATCH 14/18] add java doc --- .../java/org/apache/hadoop/ozone/OzoneSecurityUtil.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java index 10fbd150cae8..21e6ffbb4100 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java @@ -125,6 +125,12 @@ public static List getValidInetsForCurrentHost() return hostIps; } + /** + * Convert list of string encoded certificates to list of X509Certificate. + * @param pemEncodedCerts + * @return list of X509Certificate. + * @throws IOException + */ public static List convertToX509( List pemEncodedCerts) throws IOException { List x509Certificates = From 0b42164f14b60c97f04c0d90d6f298ca01f95f7d Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 29 Mar 2021 14:42:45 +0530 Subject: [PATCH 15/18] make ca list full ca list in om/dn --- .../transport/server/ratis/XceiverServerRatis.java | 14 ++++++-------- .../ozone/om/ratis/OzoneManagerRatisServer.java | 10 ++++------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 8f779a59e11e..6fd2706a8b34 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.tracing.TracingUtil; +import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -462,24 +463,21 @@ public static XceiverServerRatis newXceiverServerRatis( // DN Ratis server act as both SSL client and server and we must pass TLS // configuration for both. private static Parameters createTlsParameters(SecurityConfig conf, - CertificateClient caClient) { + CertificateClient caClient) throws IOException { Parameters parameters = new Parameters(); if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { - List listCA = new ArrayList<>(); - listCA.add(caClient.getCACertificate()); - if (caClient.getRootCACertificate() != null) { - listCA.add(caClient.getRootCACertificate()); - } + List caList = HAUtils.buildCAX509List(caClient, + conf.getConfiguration()); GrpcTlsConfig serverConfig = new GrpcTlsConfig( caClient.getPrivateKey(), caClient.getCertificate(), - listCA, true); + caList, true); GrpcConfigKeys.Server.setTlsConf(parameters, serverConfig); GrpcConfigKeys.Admin.setTlsConf(parameters, serverConfig); GrpcTlsConfig clientConfig = new GrpcTlsConfig( caClient.getPrivateKey(), caClient.getCertificate(), - listCA, false); + caList, false); GrpcConfigKeys.Client.setTlsConf(parameters, clientConfig); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java index 5a76e0714a4b..02e5e185e499 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.tracing.TracingUtil; +import org.apache.hadoop.hdds.utils.HAUtils; import org.apache.hadoop.ipc.ProtobufRpcEngine.Server; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; @@ -652,14 +653,11 @@ private static Parameters createServerTlsParameters(SecurityConfig conf, Parameters parameters = new Parameters(); if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) { - List listCA = new ArrayList<>(); - listCA.add(caClient.getCACertificate()); - if (caClient.getRootCACertificate() != null) { - listCA.add(caClient.getRootCACertificate()); - } + List caList = HAUtils.buildCAX509List(caClient, + conf.getConfiguration()); GrpcTlsConfig config = new GrpcTlsConfig( caClient.getPrivateKey(), caClient.getCertificate(), - listCA, true); + caList, true); GrpcConfigKeys.Server.setTlsConf(parameters, config); GrpcConfigKeys.Admin.setTlsConf(parameters, config); GrpcConfigKeys.Client.setTlsConf(parameters, config); From 76d6a86a1fdb74f656a4806f9efe510b310a2f03 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Wed, 31 Mar 2021 12:18:07 +0530 Subject: [PATCH 16/18] fix review comments and few more changes found during addressing review comments --- .../certificate/client/CertificateClient.java | 13 +++ .../client/DefaultCertificateClient.java | 10 ++ .../org/apache/hadoop/hdds/utils/HAUtils.java | 101 ++++++++++-------- .../scm/server/SCMSecurityProtocolServer.java | 16 ++- .../scm/server/StorageContainerManager.java | 82 ++++++++------ .../server/TestSCMSecurityProtocolServer.java | 2 +- .../client/CertificateClientTestImpl.java | 4 + 7 files changed, 144 insertions(+), 84 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java index 75bc98322527..fb51227ff214 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java @@ -234,8 +234,21 @@ enum InitResponse { void storeRootCACertificate(String pemEncodedCert, boolean force) throws CertificateException; + /** + * Return the pem encoded CA certificate list. + * + * If initialized return list of pem encoded CA certificates, else return + * null. + * @return list of pem encoded CA certificates. + */ + List getCAList(); + /** * Return the pem encoded CA certificate list. + * + * If list is null, fetch the list from SCM and returns the list. + * If list is not null, return the pem encoded CA certificate list. + * * @return list of pem encoded CA certificates. * @throws IOException */ diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index 00244eeb5f59..19c0e78ce12d 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -871,6 +871,16 @@ public void storeRootCACertificate(String pemEncodedCert, boolean force) } } + @Override + public List getCAList() { + lock.lock(); + try { + return pemEncodedCACerts; + } finally { + lock.unlock(); + } + } + @Override public List listCA() throws IOException { lock.lock(); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java index 212250f936a9..db129f43748c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java @@ -20,7 +20,7 @@ import com.google.protobuf.ServiceException; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; +import org.apache.hadoop.hdds.function.SupplierWithIOException; import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.AddSCMRequest; @@ -365,10 +365,7 @@ public static void checkSecurityAndSCMHAEnabled(OzoneConfiguration conf) { public static List buildCAList(CertificateClient certClient, ConfigurationSource configuration) throws IOException { //TODO: make it configurable. - long waitTime = 5 * 60 * 1000L; - long retryTime = 10 * 1000L; - long currentTime = Time.monotonicNow(); - List caCertPemList = null; + List caCertPemList; if (certClient != null) { caCertPemList = new ArrayList<>(); if (!SCMHAUtils.isSCMHAEnabled(configuration)) { @@ -379,37 +376,28 @@ public static List buildCAList(CertificateClient certClient, caCertPemList.add(CertificateCodec.getPEMEncodedString( certClient.getCACertificate())); } else { - // TODO: If SCMs are bootstrapped later, then listCA need to be - // refetched if listCA size is less than scm ha config node list size. - // For now when Client of SCM's are started we compare their node list - // size and ca list size if it is as expected, we return the ca list. - boolean caListUpToDate; Collection scmNodes = SCMHAUtils.getSCMNodeIds(configuration); - // TODO: make them configurable. + int expectedCount = scmNodes.size() + 1; if (scmNodes.size() > 1) { - do { - caCertPemList = certClient.updateCAList(); - caListUpToDate = - caCertPemList.size() == scmNodes.size() + 1 ? true : false; - if (!caListUpToDate) { - try { - Thread.sleep(retryTime); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - } - } while (!caListUpToDate && - Time.monotonicNow() - currentTime < waitTime); - checkCertCount(caCertPemList.size(), scmNodes.size() + 1); + // First check if cert client has ca list initialized. + // This is being done, when this method is called multiple times we + // don't make call to SCM, we return from in-memory. + caCertPemList = certClient.getCAList(); + if (caCertPemList != null && caCertPemList.size() == expectedCount) { + return caCertPemList; + } + caCertPemList = waitForCACerts(() -> certClient.updateCAList(), + expectedCount); + checkCertCount(caCertPemList.size(), expectedCount); } else { - caCertPemList = certClient.updateCAList(); + caCertPemList = certClient.listCA(); } } } else { + SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient = + HddsServerUtil.getScmSecurityClient(configuration); if (!SCMHAUtils.isSCMHAEnabled(configuration)) { caCertPemList = new ArrayList<>(); - SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient = - HddsServerUtil.getScmSecurityClient(configuration); SCMGetCertResponseProto scmGetCertResponseProto = scmSecurityProtocolClient.getCACert(); if (scmGetCertResponseProto.hasX509Certificate()) { @@ -420,26 +408,13 @@ public static List buildCAList(CertificateClient certClient, } } else { Collection scmNodes = SCMHAUtils.getSCMNodeIds(configuration); - SCMSecurityProtocol scmSecurityProtocolClient = - HddsServerUtil.getScmSecurityClient(configuration); - boolean caListUpToDate; + int expectedCount = scmNodes.size() + 1; if (scmNodes.size() > 1) { - do { - caCertPemList = - scmSecurityProtocolClient.listCACertificate(); - caListUpToDate = - caCertPemList.size() == scmNodes.size() + 1 ? true : false; - if (!caListUpToDate) { - try { - Thread.sleep(retryTime); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } - } - } while (!caListUpToDate && - Time.monotonicNow() - currentTime < waitTime); - checkCertCount(caCertPemList.size(), scmNodes.size() + 1); - } else { + caCertPemList = waitForCACerts( + () -> scmSecurityProtocolClient.listCACertificate(), + expectedCount); + checkCertCount(caCertPemList.size(), expectedCount); + } else{ caCertPemList = scmSecurityProtocolClient.listCACertificate(); } } @@ -447,6 +422,38 @@ public static List buildCAList(CertificateClient certClient, return caCertPemList; } + private static List waitForCACerts( + final SupplierWithIOException> applyFunction, + int expectedCount) throws IOException { + //TODO: make wait time and sleep time configurable if needed. + // TODO: If SCMs are bootstrapped later, then listCA need to be + // refetched if listCA size is less than scm ha config node list size. + // For now when Client of SCM's are started we compare their node list + // size and ca list size if it is as expected, we return the ca list. + boolean caListUpToDate; + long waitTime = 5 * 60 * 1000L; + long retryTime = 10 * 1000L; + long currentTime = Time.monotonicNow(); + List caCertPemList; + do { + caCertPemList = applyFunction.get(); + caListUpToDate = + caCertPemList.size() == expectedCount ? true : false; + if (!caListUpToDate) { + LOG.info("Expected CA list size {}, where as received CA List size " + + "{}. Retry to fetch CA List after {} seconds", expectedCount, + caCertPemList.size(), waitTime/1000); + try { + Thread.sleep(retryTime); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + } while (!caListUpToDate && + Time.monotonicNow() - currentTime < waitTime); + return caCertPemList; + } + private static void checkCertCount(int certCount, int expectedCount) throws SCMSecurityException{ diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java index 5c8642d0b824..e3fd23ea2017 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java @@ -72,6 +72,7 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol { .getLogger(SCMSecurityProtocolServer.class); private final CertificateServer rootCertificateServer; private final CertificateServer scmCertificateServer; + private final X509Certificate rootCACertificate; private final RPC.Server rpcServer; private final InetSocketAddress rpcAddress; private final ProtocolMessageMetrics metrics; @@ -79,11 +80,13 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol { SCMSecurityProtocolServer(OzoneConfiguration conf, CertificateServer rootCertificateServer, - CertificateServer scmCertificateServer, StorageContainerManager scm) + CertificateServer scmCertificateServer, + X509Certificate rootCACert, StorageContainerManager scm) throws IOException { this.storageContainerManager = scm; this.rootCertificateServer = rootCertificateServer; this.scmCertificateServer = scmCertificateServer; + this.rootCACertificate = rootCACert; final int handlerCount = conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY, ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT); @@ -314,8 +317,7 @@ public String getRootCACertificate() throws IOException { LOGGER.debug("Getting Root CA certificate."); if (storageContainerManager.getScmStorageConfig() .getPrimaryScmNodeId() != null) { - return CertificateCodec.getPEMEncodedString( - storageContainerManager.getScmCertificateClient().getCACertificate()); + return CertificateCodec.getPEMEncodedString(rootCACertificate); } return null; } @@ -351,4 +353,12 @@ public void join() throws InterruptedException { getRpcServer().join(); } + public CertificateServer getRootCertificateServer() { + return rootCertificateServer; + } + + + public CertificateServer getScmCertificateServer() { + return scmCertificateServer; + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 72892e7bc6a9..bfd45d8ed36d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -220,8 +220,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private final LeaseManager commandWatcherLeaseManager; private SCMSafeModeManager scmSafeModeManager; - private CertificateServer rootCertificateServer; - private CertificateServer scmCertificateServer; private SCMCertificateClient scmCertificateClient; private JvmPauseMonitor jvmPauseMonitor; @@ -329,8 +327,6 @@ private StorageContainerManager(OzoneConfiguration conf, // if no Security, we do not create a Certificate Server at all. // This allows user to boot SCM without security temporarily // and then come back and enable it without any impact. - rootCertificateServer = null; - scmCertificateServer = null; securityProtocolServer = null; } @@ -578,6 +574,7 @@ private void initializeSystemManagers(OzoneConfiguration conf, private void initializeCAnSecurityProtocol(OzoneConfiguration conf, SCMConfigurator configurator) throws IOException { + // TODO: Support Certificate Server loading via Class Name loader. // So it is easy to use different Certificate Servers if needed. if(this.scmMetadataStore == null) { @@ -593,6 +590,8 @@ private void initializeCAnSecurityProtocol(OzoneConfiguration conf, .setCRLSequenceId(getLastSequenceIdForCRL()).build(); + final CertificateServer scmCertificateServer; + final CertificateServer rootCertificateServer; // If primary SCM node Id is set it means this is a cluster which has // performed init with SCM HA version code. if (scmStorageConfig.checkPrimarySCMIdInitialized()) { @@ -600,7 +599,7 @@ private void initializeCAnSecurityProtocol(OzoneConfiguration conf, String subject = SCM_SUB_CA_PREFIX + InetAddress.getLocalHost().getHostName(); if (configurator.getCertificateServer() != null) { - this.scmCertificateServer = configurator.getCertificateServer(); + scmCertificateServer = configurator.getCertificateServer(); } else { scmCertificateServer = new DefaultCAServer(subject, scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(), @@ -613,33 +612,13 @@ certificateStore, new DefaultProfile(), if (primaryScmNodeId.equals(scmStorageConfig.getScmId())) { if (configurator.getCertificateServer() != null) { - this.rootCertificateServer = configurator.getCertificateServer(); + rootCertificateServer = configurator.getCertificateServer(); } else { rootCertificateServer = HASecurityUtils.initializeRootCertificateServer( conf, certificateStore, scmStorageConfig); } - - BigInteger certSerial = - scmCertificateClient.getCertificate().getSerialNumber(); - // Store the certificate in DB. On primary SCM when init happens, the - // certificate is not persisted to DB. As we don't have Metadatstore - // and ratis server initialized with statemachine. We need to do only - // for primary scm, for other bootstrapped scm's certificates will be - // persisted via ratis. - if (certificateStore.getCertificateByID(certSerial, - VALID_CERTS) == null) { - LOG.info("Storing certSerial {}", certSerial); - certificateStore.storeValidScmCertificate( - certSerial, scmCertificateClient.getCertificate()); - } - X509Certificate rootCACert = scmCertificateClient.getCACertificate(); - if (certificateStore.getCertificateByID(rootCACert.getSerialNumber(), - VALID_CERTS) == null) { - LOG.info("Storing root certSerial {}", rootCACert.getSerialNumber()); - certificateStore.storeValidScmCertificate( - rootCACert.getSerialNumber(), rootCACert); - } + persistPrimarySCMCerts(); } else { rootCertificateServer = null; } @@ -654,16 +633,48 @@ certificateStore, new DefaultProfile(), scmCertificateServer = rootCertificateServer; } + // We need to pass getCACertificate as rootCA certificate, + // as for SCM CA is root-CA. securityProtocolServer = new SCMSecurityProtocolServer(conf, - rootCertificateServer, scmCertificateServer, this); + rootCertificateServer, scmCertificateServer, + scmCertificateClient.getCACertificate(), this); + } + + /** Persist primary SCM root ca cert and sub-ca certs to DB. + * + * @throws IOException + */ + private void persistPrimarySCMCerts() throws IOException { + BigInteger certSerial = + scmCertificateClient.getCertificate().getSerialNumber(); + // Store the certificate in DB. On primary SCM when init happens, the + // certificate is not persisted to DB. As we don't have Metadatstore + // and ratis server initialized with statemachine. We need to do only + // for primary scm, for other bootstrapped scm's certificates will be + // persisted via ratis. + if (certificateStore.getCertificateByID(certSerial, + VALID_CERTS) == null) { + LOG.info("Storing sub-ca certificate serialId {} on primary SCM", + certSerial); + certificateStore.storeValidScmCertificate( + certSerial, scmCertificateClient.getCertificate()); + } + X509Certificate rootCACert = scmCertificateClient.getCACertificate(); + if (certificateStore.getCertificateByID(rootCACert.getSerialNumber(), + VALID_CERTS) == null) { + LOG.info("Storing root certificate serialId {}", + rootCACert.getSerialNumber()); + certificateStore.storeValidScmCertificate( + rootCACert.getSerialNumber(), rootCACert); + } } public CertificateServer getRootCertificateServer() { - return rootCertificateServer; + return getSecurityProtocolServer().getRootCertificateServer(); } public CertificateServer getScmCertificateServer() { - return scmCertificateServer; + return getSecurityProtocolServer().getScmCertificateServer(); } public SCMCertificateClient getScmCertificateClient() { @@ -1142,6 +1153,10 @@ public void start() throws IOException { setStartTime(); } + /** Persist SCM certs to DB on bootstrap scm nodes. + * + * @throws IOException + */ private void persistSCMCertificates() throws IOException { // Fetch all CA's and persist during startup on bootstrap nodes. This // is primarily being done to persist primary SCM Cert and Root CA. @@ -1149,7 +1164,7 @@ private void persistSCMCertificates() throws IOException { if (primaryScmNodeId != null && !primaryScmNodeId.equals( scmStorageConfig.getScmId())) { List pemEncodedCerts = - scmCertificateClient.updateCAList(); + scmCertificateClient.listCA(); // Write the primary SCM CA and Root CA during startup. for (String cert : pemEncodedCerts) { @@ -1158,8 +1173,9 @@ private void persistSCMCertificates() throws IOException { CertificateCodec.getX509Certificate(cert); if (certificateStore.getCertificateByID( x509Certificate.getSerialNumber(), VALID_CERTS) == null) { - LOG.info("Persist certserial {} on Scm Bootstrap Node {}", - x509Certificate.getSerialNumber(), scmStorageConfig.getScmId()); + LOG.info("Persist certificate serialId {} on Scm Bootstrap Node " + + "{}", x509Certificate.getSerialNumber(), + scmStorageConfig.getScmId()); certificateStore.storeValidScmCertificate( x509Certificate.getSerialNumber(), x509Certificate); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java index 4fa1d4636b8a..86a8c3e61235 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java @@ -42,7 +42,7 @@ public void setUp() throws Exception { config.set(OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY, OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT + ":0"); securityProtocolServer = new SCMSecurityProtocolServer(config, null, - null, null); + null, null, null); } @After diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java index 0f487c668fe0..97ecdd5ffba4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java @@ -199,6 +199,10 @@ public void storeRootCACertificate(String pemEncodedCert, boolean force) { } + @Override + public List getCAList() { + return null; + } @Override public List listCA() throws IOException { return null; From 6868a29172eba25af30315f33e2f2b59a6bfeb79 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Thu, 1 Apr 2021 07:48:52 +0530 Subject: [PATCH 17/18] add wait for scm similar to secure-ha --- .../dist/src/main/compose/ozone-ha/docker-compose.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml index 48adc786f9f7..4489a5530d41 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-compose.yaml @@ -41,6 +41,7 @@ services: om1: <<: *common-config environment: + WAITFOR: scm3:9865 ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION <<: *replication ports: @@ -51,6 +52,7 @@ services: om2: <<: *common-config environment: + WAITFOR: scm3:9865 ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION <<: *replication ports: @@ -61,6 +63,7 @@ services: om3: <<: *common-config environment: + WAITFOR: scm3:9865 ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION <<: *replication ports: From 39a855bbe1493ab3e7bb6c29ee9b5a85377fb728 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Thu, 1 Apr 2021 08:55:37 +0530 Subject: [PATCH 18/18] Trigger Build