From e84b5e2e6cd6e06fdc30c2b5309915700734b029 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 27 Apr 2021 12:30:34 +0530 Subject: [PATCH 1/3] HDDS-5060. [SCM HA Security] Make InterSCM grpc channel secure. --- .../hdds/scm/ha/InterSCMGrpcClient.java | 29 ++++++++++++++++--- .../scm/ha/InterSCMGrpcProtocolService.java | 23 +++++++++++++++ .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 3 +- .../hdds/scm/ha/SCMSnapshotProvider.java | 12 ++++++-- 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java index 08dd307c1b2c..c95f647fb33f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java @@ -22,10 +22,15 @@ import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolProtos; import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolProtos.CopyDBCheckpointResponseProto; import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolServiceGrpc; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ratis.thirdparty.io.grpc.ManagedChannel; +import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts; import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder; import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; +import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,15 +55,31 @@ public class InterSCMGrpcClient implements SCMSnapshotDownloader{ private final InterSCMProtocolServiceGrpc.InterSCMProtocolServiceStub client; + private final long timeout; - public InterSCMGrpcClient(final String host, final int leaderPort, - final ConfigurationSource conf) { - final int port = leaderPort; - final long timeout = + public InterSCMGrpcClient(final String host, + int port, final ConfigurationSource conf, + SCMCertificateClient scmCertificateClient) throws IOException { + Preconditions.checkNotNull(conf); + timeout = conf.getObject(SCMHAConfiguration.class).getGrpcDeadlineInterval(); NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(host, port).usePlaintext() .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE); + SecurityConfig securityConfig = new SecurityConfig(conf); + if (securityConfig.isSecurityEnabled() + && securityConfig.isGrpcTlsEnabled()) { + SslContextBuilder sslClientContextBuilder = SslContextBuilder.forClient(); + sslClientContextBuilder.keyManager(scmCertificateClient.getPrivateKey(), + scmCertificateClient.getCertificate()); + sslClientContextBuilder.trustManager( + scmCertificateClient.getCACertificate()); + SslContextBuilder sslContextBuilder = GrpcSslContexts.configure( + sslClientContextBuilder, securityConfig.getGrpcSslProvider()); + channelBuilder.sslContext(sslContextBuilder.build()) + .useTransportSecurity(); + } + channel = channelBuilder.build(); client = InterSCMProtocolServiceGrpc.newStub(channel). withDeadlineAfter(timeout, TimeUnit.SECONDS); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java index b6f08a5fcdf2..41ec803aaaab 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java @@ -25,10 +25,14 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.security.exception.SCMSecurityException; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ratis.thirdparty.io.grpc.Server; import org.apache.ratis.thirdparty.io.grpc.ServerBuilder; +import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts; import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder; +import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,6 +60,25 @@ public InterSCMGrpcProtocolService(final ConfigurationSource conf, InterSCMGrpcService service = new InterSCMGrpcService(scm); ServerBuilder b = nettyServerBuilder.addService(service); + + SecurityConfig securityConfig = new SecurityConfig(conf); + if (securityConfig.isSecurityEnabled() + && securityConfig.isGrpcTlsEnabled()) { + try { + SslContextBuilder sslServerContextBuilder = + SslContextBuilder.forServer( + scm.getScmCertificateClient().getPrivateKey(), + scm.getScmCertificateClient().getCertificate()); + SslContextBuilder sslContextBuilder = GrpcSslContexts.configure( + sslServerContextBuilder, securityConfig.getGrpcSslProvider()); + nettyServerBuilder.sslContext(sslContextBuilder.build()); + } catch (Exception ex) { + LOG.error("Unable to setup TLS for secure InterSCMGrpcProtocolService " + + "GRPC endpoint.", ex); + throw new RuntimeException("Unable to setup TLS for secure " + + "InterSCMGrpcProtocolService GRPC endpoint."); + } + } Preconditions.checkNotNull(b); server = nettyServerBuilder.build(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java index 00aff5ec8f71..8fb305038a84 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java @@ -78,7 +78,8 @@ public SCMHAManagerImpl(final ConfigurationSource conf, this.ratisServer = new SCMRatisServerImpl(conf, scm, (SCMHADBTransactionBuffer) transactionBuffer); this.scmSnapshotProvider = new SCMSnapshotProvider(conf, - scm.getSCMHANodeDetails().getPeerNodeDetails()); + scm.getSCMHANodeDetails().getPeerNodeDetails(), + scm.getScmCertificateClient()); grpcServer = new InterSCMGrpcProtocolService(conf, scm); } else { this.transactionBuffer = new SCMDBTransactionBufferImpl(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java index e5bdfbe38849..a6f459e542a9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.cert.X509Certificate; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,6 +33,8 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.RocksDBCheckpoint; @@ -57,10 +60,14 @@ public class SCMSnapshotProvider { private Map peerNodesMap; + private final SCMCertificateClient scmCertificateClient; + public SCMSnapshotProvider(ConfigurationSource conf, - List peerNodes) { + List peerNodes, + SCMCertificateClient scmCertificateClient) { LOG.info("Initializing SCM Snapshot Provider"); this.conf = conf; + this.scmCertificateClient = scmCertificateClient; // Create Ratis storage dir String scmRatisDirectory = SCMHAUtils.getSCMRatisDirectory(conf); @@ -101,12 +108,13 @@ public DBCheckpoint getSCMDBSnapshot(String leaderSCMNodeID) .getAbsolutePath(); File targetFile = new File(snapshotFilePath + ".tar.gz"); + // the downloadClient instance will be created as and when install snapshot // request is received. No caching of the client as it should be a very rare int port = peerNodesMap.get(leaderSCMNodeID).getGrpcPort(); SCMSnapshotDownloader downloadClient = new InterSCMGrpcClient( peerNodesMap.get(leaderSCMNodeID).getInetAddress().getHostAddress(), - port, conf); + port, conf, scmCertificateClient); try { downloadClient.download(targetFile.toPath()).get(); } catch (ExecutionException | InterruptedException e) { From ea94478e629b89ccc6375fba73ef4948bcd0b68e Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 27 Apr 2021 13:09:13 +0530 Subject: [PATCH 2/3] fix cs and initialize cert client only on secure cluster --- .../org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 1 - .../hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java | 5 ++--- .../org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 2 -- .../hadoop/hdds/scm/server/StorageContainerManager.java | 3 ++- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java index c95f647fb33f..2bb840c93bd3 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcClient.java @@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolProtos; import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolProtos.CopyDBCheckpointResponseProto; import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolServiceGrpc; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; import org.apache.hadoop.ozone.OzoneConsts; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java index 41ec803aaaab..b92f4e43a3e5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; -import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ratis.thirdparty.io.grpc.Server; @@ -73,8 +72,8 @@ public InterSCMGrpcProtocolService(final ConfigurationSource conf, sslServerContextBuilder, securityConfig.getGrpcSslProvider()); nettyServerBuilder.sslContext(sslContextBuilder.build()); } catch (Exception ex) { - LOG.error("Unable to setup TLS for secure InterSCMGrpcProtocolService " + - "GRPC endpoint.", ex); + LOG.error("Unable to setup TLS for secure " + + "InterSCMGrpcProtocolService GRPC endpoint.", ex); throw new RuntimeException("Unable to setup TLS for secure " + "InterSCMGrpcProtocolService GRPC endpoint."); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java index a6f459e542a9..631bc75011f5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.cert.X509Certificate; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -33,7 +32,6 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.RocksDBCheckpoint; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 13ee92161a1d..453d421b48e2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -413,7 +413,8 @@ private StorageContainerManager(OzoneConfiguration conf, } private void initializeCertificateClient() { - if (scmStorageConfig.checkPrimarySCMIdInitialized()) { + if (OzoneSecurityUtil.isSecurityEnabled(configuration) && + scmStorageConfig.checkPrimarySCMIdInitialized()) { scmCertificateClient = new SCMCertificateClient( new SecurityConfig(configuration), scmStorageConfig.getScmCertSerialId()); From 5ec1d08bb707c7d77593e566842bb3c6dc33cd69 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 27 Apr 2021 16:10:48 +0530 Subject: [PATCH 3/3] trigger ci