From c0775f7ba07608befc840ec3079d0443d0524d1c Mon Sep 17 00:00:00 2001 From: marsishandsome Date: Wed, 22 Sep 2021 16:53:40 +0800 Subject: [PATCH 1/4] support tls Signed-off-by: marsishandsome --- README.md | 18 +++++ pom.xml | 6 ++ .../java/org/tikv/common/ConfigUtils.java | 6 ++ .../java/org/tikv/common/TiConfiguration.java | 38 ++++++++++ src/main/java/org/tikv/common/TiSession.java | 13 +++- .../org/tikv/common/util/ChannelFactory.java | 75 +++++++++++++++++-- 6 files changed, 146 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 895d72dd3fe..0e4544eb4ab 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,24 @@ The following includes ThreadPool related parameters, which can be passed in thr - whether to enable `Compare And Set`, set true if using `RawKVClient.compareAndSet` or `RawKVClient.putIfAbsent` - default: false +### TLS + +#### tikv.tls_enable +- whether to enable tls +- default: false + +#### tikv.trust_cert_collection +- Trusted certificates for verifying the remote endpoint's certificate, e.g. /home/tidb/ca.pem. The file should contain an X.509 certificate collection in PEM format. +- default: null + +#### tikv.key_cert_chain +- an X.509 certificate chain file in PEM format, e.g. /home/tidb/client.pem. +- default: null + +#### tikv.key_file +- a PKCS#8 private key file in PEM format. e.g. /home/tidb/client-key.pem. +- default: null + ## Metrics Client Java supports exporting metrics to Prometheus using poll mode and viewing on Grafana. The following steps shows how to enable this function. diff --git a/pom.xml b/pom.xml index 09b8374f63b..034c8e69219 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,7 @@ 1.2.17 1.7.16 1.24.0 + 2.0.25.Final 1.6.6 2.12.3 3.0.1 @@ -133,6 +134,11 @@ grpc-services ${grpc.version} + + io.netty + netty-tcnative-boringssl-static + ${netty.tcnative.version} + io.grpc grpc-testing diff --git a/src/main/java/org/tikv/common/ConfigUtils.java b/src/main/java/org/tikv/common/ConfigUtils.java index de50e7c8702..8374d6feb59 100644 --- a/src/main/java/org/tikv/common/ConfigUtils.java +++ b/src/main/java/org/tikv/common/ConfigUtils.java @@ -69,6 +69,11 @@ public class ConfigUtils { public static final String TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS = "tikv.rawkv.default_backoff_in_ms"; + public static final String TIKV_TLS_ENABLE = "tikv.tls_enable"; + public static final String TIKV_TRUST_CERT_COLLECTION = "tikv.trust_cert_collection"; + public static final String TIKV_KEY_CERT_CHAIN = "tikv.key_cert_chain"; + public static final String TIKV_KEY_FILE = "tikv.key_file"; + public static final String DEF_PD_ADDRESSES = "127.0.0.1:2379"; public static final String DEF_TIMEOUT = "200ms"; public static final String DEF_TIKV_GRPC_INGEST_TIMEOUT = "200s"; @@ -125,4 +130,5 @@ public class ConfigUtils { public static final int DEF_TIKV_GRPC_KEEPALIVE_TIME = 10; public static final int DEF_TIKV_GRPC_KEEPALIVE_TIMEOUT = 3; + public static final boolean DEF_TIKV_TLS_ENABLE = false; } diff --git a/src/main/java/org/tikv/common/TiConfiguration.java b/src/main/java/org/tikv/common/TiConfiguration.java index eb3e64d80e4..311313e727a 100644 --- a/src/main/java/org/tikv/common/TiConfiguration.java +++ b/src/main/java/org/tikv/common/TiConfiguration.java @@ -90,6 +90,7 @@ private static void loadFromDefaultProperties() { setIfMissing(TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS, DEF_TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS); setIfMissing(TIKV_GRPC_KEEPALIVE_TIME, DEF_TIKV_GRPC_KEEPALIVE_TIME); setIfMissing(TIKV_GRPC_KEEPALIVE_TIMEOUT, DEF_TIKV_GRPC_KEEPALIVE_TIMEOUT); + setIfMissing(TIKV_TLS_ENABLE, DEF_TIKV_TLS_ENABLE); } public static void listAll() { @@ -291,6 +292,11 @@ private static ReplicaRead getReplicaRead(String key) { private int rawKVDefaultBackoffInMS = getInt(TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS); + private boolean tlsEnable = getBoolean(TIKV_TLS_ENABLE); + private Optional trustCertCollectionFile = getOption(TIKV_TRUST_CERT_COLLECTION); + private Optional keyCertChainFile = getOption(TIKV_KEY_CERT_CHAIN); + private Optional keyFile = getOption(TIKV_KEY_FILE); + private boolean isTest = false; private int keepaliveTime = getInt(TIKV_GRPC_KEEPALIVE_TIME); @@ -689,4 +695,36 @@ public int getKeepaliveTimeout() { public void setKeepaliveTimeout(int timeout) { this.keepaliveTimeout = timeout; } + + public boolean isTlsEnable() { + return tlsEnable; + } + + public void setTlsEnable(boolean tlsEnable) { + this.tlsEnable = tlsEnable; + } + + public Optional getTrustCertCollectionFile() { + return trustCertCollectionFile; + } + + public void setTrustCertCollectionFile(Optional trustCertCollectionFile) { + this.trustCertCollectionFile = trustCertCollectionFile; + } + + public Optional getKeyCertChainFile() { + return keyCertChainFile; + } + + public void setKeyCertChainFile(Optional keyCertChainFile) { + this.keyCertChainFile = keyCertChainFile; + } + + public Optional getKeyFile() { + return keyFile; + } + + public void setKeyFile(Optional keyFile) { + this.keyFile = keyFile; + } } diff --git a/src/main/java/org/tikv/common/TiSession.java b/src/main/java/org/tikv/common/TiSession.java index 3866cb50e28..449d8765645 100644 --- a/src/main/java/org/tikv/common/TiSession.java +++ b/src/main/java/org/tikv/common/TiSession.java @@ -79,8 +79,17 @@ public class TiSession implements AutoCloseable { public TiSession(TiConfiguration conf) { this.conf = conf; this.channelFactory = - new ChannelFactory( - conf.getMaxFrameSize(), conf.getKeepaliveTime(), conf.getKeepaliveTimeout()); + conf.isTlsEnable() + ? new ChannelFactory( + conf.getMaxFrameSize(), + conf.getKeepaliveTime(), + conf.getKeepaliveTimeout(), + conf.getTrustCertCollectionFile(), + conf.getKeyCertChainFile(), + conf.getKeyFile()) + : new ChannelFactory( + conf.getMaxFrameSize(), conf.getKeepaliveTime(), conf.getKeepaliveTimeout()); + this.client = PDClient.createRaw(conf, channelFactory); this.enableGrpcForward = conf.getEnableGrpcForward(); this.metricsServer = MetricsServer.getInstance(conf); diff --git a/src/main/java/org/tikv/common/util/ChannelFactory.java b/src/main/java/org/tikv/common/util/ChannelFactory.java index 5433ad5acd3..533b978283e 100644 --- a/src/main/java/org/tikv/common/util/ChannelFactory.java +++ b/src/main/java/org/tikv/common/util/ChannelFactory.java @@ -17,22 +17,63 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.NettyChannelBuilder; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import java.io.File; import java.net.URI; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import javax.net.ssl.SSLException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tikv.common.HostMapping; import org.tikv.common.pd.PDUtils; public class ChannelFactory implements AutoCloseable { + private static final Logger logger = LoggerFactory.getLogger(ChannelFactory.class); + private final int maxFrameSize; private final int keepaliveTime; private final int keepaliveTimeout; private final ConcurrentHashMap connPool = new ConcurrentHashMap<>(); + private final SslContextBuilder sslContextBuilder; public ChannelFactory(int maxFrameSize, int keepaliveTime, int keepaliveTimeout) { this.maxFrameSize = maxFrameSize; this.keepaliveTime = keepaliveTime; this.keepaliveTimeout = keepaliveTimeout; + this.sslContextBuilder = null; + } + + public ChannelFactory( + int maxFrameSize, + int keepaliveTime, + int keepaliveTimeout, + Optional trustCertCollectionFilePath, + Optional keyCertChainFilePath, + Optional keyFilePath) { + this.maxFrameSize = maxFrameSize; + this.keepaliveTime = keepaliveTime; + this.keepaliveTimeout = keepaliveTimeout; + this.sslContextBuilder = + getSslContextBuilder(trustCertCollectionFilePath, keyCertChainFilePath, keyFilePath); + } + + private SslContextBuilder getSslContextBuilder( + Optional trustCertCollectionFilePath, + Optional keyCertChainFilePath, + Optional keyFilePath) { + SslContextBuilder builder = GrpcSslContexts.forClient(); + if (trustCertCollectionFilePath.isPresent()) { + builder.trustManager(new File(trustCertCollectionFilePath.get())); + } + if (keyCertChainFilePath.isPresent() && keyFilePath.isPresent()) { + builder.keyManager(new File(keyCertChainFilePath.get()), new File(keyFilePath.get())); + } + return builder; } public ManagedChannel getChannel(String addressStr, HostMapping hostMapping) { @@ -53,14 +94,32 @@ public ManagedChannel getChannel(String addressStr, HostMapping hostMapping) { } // Channel should be lazy without actual connection until first call // So a coarse grain lock is ok here - return ManagedChannelBuilder.forAddress(mappedAddr.getHost(), mappedAddr.getPort()) - .maxInboundMessageSize(maxFrameSize) - .keepAliveTime(keepaliveTime, TimeUnit.SECONDS) - .keepAliveTimeout(keepaliveTimeout, TimeUnit.SECONDS) - .keepAliveWithoutCalls(true) - .usePlaintext(true) - .idleTimeout(60, TimeUnit.SECONDS) - .build(); + if (sslContextBuilder == null) { + return ManagedChannelBuilder.forAddress(mappedAddr.getHost(), mappedAddr.getPort()) + .maxInboundMessageSize(maxFrameSize) + .keepAliveTime(keepaliveTime, TimeUnit.SECONDS) + .keepAliveTimeout(keepaliveTimeout, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .usePlaintext(true) + .idleTimeout(60, TimeUnit.SECONDS) + .build(); + } else { + SslContext sslContext = null; + try { + sslContext = sslContextBuilder.build(); + } catch (SSLException e) { + logger.error("create ssl context failed!", e); + return null; + } + return NettyChannelBuilder.forAddress(mappedAddr.getHost(), mappedAddr.getPort()) + .maxInboundMessageSize(maxFrameSize) + .keepAliveTime(keepaliveTime, TimeUnit.SECONDS) + .keepAliveTimeout(keepaliveTimeout, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .sslContext(sslContext) + .idleTimeout(60, TimeUnit.SECONDS) + .build(); + } }); } From d7b60c02432fa59bc57efee05a5534cb1ef47497 Mon Sep 17 00:00:00 2001 From: marsishandsome Date: Fri, 24 Sep 2021 15:52:27 +0800 Subject: [PATCH 2/4] address code review Signed-off-by: marsishandsome --- .../org/tikv/common/util/ChannelFactory.java | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/tikv/common/util/ChannelFactory.java b/src/main/java/org/tikv/common/util/ChannelFactory.java index 533b978283e..de0052e96cf 100644 --- a/src/main/java/org/tikv/common/util/ChannelFactory.java +++ b/src/main/java/org/tikv/common/util/ChannelFactory.java @@ -16,7 +16,6 @@ package org.tikv.common.util; import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; import io.grpc.netty.GrpcSslContexts; import io.grpc.netty.NettyChannelBuilder; import io.netty.handler.ssl.SslContext; @@ -92,17 +91,19 @@ public ManagedChannel getChannel(String addressStr, HostMapping hostMapping) { } catch (Exception e) { throw new IllegalArgumentException("failed to get mapped address " + address, e); } + // Channel should be lazy without actual connection until first call // So a coarse grain lock is ok here + NettyChannelBuilder builder = + NettyChannelBuilder.forAddress(mappedAddr.getHost(), mappedAddr.getPort()) + .maxInboundMessageSize(maxFrameSize) + .keepAliveTime(keepaliveTime, TimeUnit.SECONDS) + .keepAliveTimeout(keepaliveTimeout, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .idleTimeout(60, TimeUnit.SECONDS); + if (sslContextBuilder == null) { - return ManagedChannelBuilder.forAddress(mappedAddr.getHost(), mappedAddr.getPort()) - .maxInboundMessageSize(maxFrameSize) - .keepAliveTime(keepaliveTime, TimeUnit.SECONDS) - .keepAliveTimeout(keepaliveTimeout, TimeUnit.SECONDS) - .keepAliveWithoutCalls(true) - .usePlaintext(true) - .idleTimeout(60, TimeUnit.SECONDS) - .build(); + return builder.usePlaintext(true).build(); } else { SslContext sslContext = null; try { @@ -111,14 +112,7 @@ public ManagedChannel getChannel(String addressStr, HostMapping hostMapping) { logger.error("create ssl context failed!", e); return null; } - return NettyChannelBuilder.forAddress(mappedAddr.getHost(), mappedAddr.getPort()) - .maxInboundMessageSize(maxFrameSize) - .keepAliveTime(keepaliveTime, TimeUnit.SECONDS) - .keepAliveTimeout(keepaliveTimeout, TimeUnit.SECONDS) - .keepAliveWithoutCalls(true) - .sslContext(sslContext) - .idleTimeout(60, TimeUnit.SECONDS) - .build(); + return builder.sslContext(sslContext).build(); } }); } From 25501d8a509ad1388e21c86a71902f72a0ffb268 Mon Sep 17 00:00:00 2001 From: marsishandsome Date: Mon, 27 Sep 2021 10:39:48 +0800 Subject: [PATCH 3/4] address code review Signed-off-by: marsishandsome --- .../java/org/tikv/common/TiConfiguration.java | 18 +++++++++--------- .../org/tikv/common/util/ChannelFactory.java | 19 ++++++++----------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/tikv/common/TiConfiguration.java b/src/main/java/org/tikv/common/TiConfiguration.java index 311313e727a..20801310887 100644 --- a/src/main/java/org/tikv/common/TiConfiguration.java +++ b/src/main/java/org/tikv/common/TiConfiguration.java @@ -293,9 +293,9 @@ private static ReplicaRead getReplicaRead(String key) { private int rawKVDefaultBackoffInMS = getInt(TIKV_RAWKV_DEFAULT_BACKOFF_IN_MS); private boolean tlsEnable = getBoolean(TIKV_TLS_ENABLE); - private Optional trustCertCollectionFile = getOption(TIKV_TRUST_CERT_COLLECTION); - private Optional keyCertChainFile = getOption(TIKV_KEY_CERT_CHAIN); - private Optional keyFile = getOption(TIKV_KEY_FILE); + private String trustCertCollectionFile = getOption(TIKV_TRUST_CERT_COLLECTION).orElse(null); + private String keyCertChainFile = getOption(TIKV_KEY_CERT_CHAIN).orElse(null); + private String keyFile = getOption(TIKV_KEY_FILE).orElse(null); private boolean isTest = false; @@ -704,27 +704,27 @@ public void setTlsEnable(boolean tlsEnable) { this.tlsEnable = tlsEnable; } - public Optional getTrustCertCollectionFile() { + public String getTrustCertCollectionFile() { return trustCertCollectionFile; } - public void setTrustCertCollectionFile(Optional trustCertCollectionFile) { + public void setTrustCertCollectionFile(String trustCertCollectionFile) { this.trustCertCollectionFile = trustCertCollectionFile; } - public Optional getKeyCertChainFile() { + public String getKeyCertChainFile() { return keyCertChainFile; } - public void setKeyCertChainFile(Optional keyCertChainFile) { + public void setKeyCertChainFile(String keyCertChainFile) { this.keyCertChainFile = keyCertChainFile; } - public Optional getKeyFile() { + public String getKeyFile() { return keyFile; } - public void setKeyFile(Optional keyFile) { + public void setKeyFile(String keyFile) { this.keyFile = keyFile; } } diff --git a/src/main/java/org/tikv/common/util/ChannelFactory.java b/src/main/java/org/tikv/common/util/ChannelFactory.java index de0052e96cf..a517473089f 100644 --- a/src/main/java/org/tikv/common/util/ChannelFactory.java +++ b/src/main/java/org/tikv/common/util/ChannelFactory.java @@ -22,7 +22,6 @@ import io.netty.handler.ssl.SslContextBuilder; import java.io.File; import java.net.URI; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLException; @@ -51,9 +50,9 @@ public ChannelFactory( int maxFrameSize, int keepaliveTime, int keepaliveTimeout, - Optional trustCertCollectionFilePath, - Optional keyCertChainFilePath, - Optional keyFilePath) { + String trustCertCollectionFilePath, + String keyCertChainFilePath, + String keyFilePath) { this.maxFrameSize = maxFrameSize; this.keepaliveTime = keepaliveTime; this.keepaliveTimeout = keepaliveTimeout; @@ -62,15 +61,13 @@ public ChannelFactory( } private SslContextBuilder getSslContextBuilder( - Optional trustCertCollectionFilePath, - Optional keyCertChainFilePath, - Optional keyFilePath) { + String trustCertCollectionFilePath, String keyCertChainFilePath, String keyFilePath) { SslContextBuilder builder = GrpcSslContexts.forClient(); - if (trustCertCollectionFilePath.isPresent()) { - builder.trustManager(new File(trustCertCollectionFilePath.get())); + if (trustCertCollectionFilePath != null) { + builder.trustManager(new File(trustCertCollectionFilePath)); } - if (keyCertChainFilePath.isPresent() && keyFilePath.isPresent()) { - builder.keyManager(new File(keyCertChainFilePath.get()), new File(keyFilePath.get())); + if (keyCertChainFilePath != null && keyFilePath != null) { + builder.keyManager(new File(keyCertChainFilePath), new File(keyFilePath)); } return builder; } From df9307680a0cca59bce8bdfe120f7c9acf0b90f5 Mon Sep 17 00:00:00 2001 From: marsishandsome Date: Mon, 27 Sep 2021 14:56:07 +0800 Subject: [PATCH 4/4] address code review Signed-off-by: marsishandsome --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e4544eb4ab..6650c1b4167 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ The following includes ThreadPool related parameters, which can be passed in thr ### TLS #### tikv.tls_enable -- whether to enable tls +- whether to enable TLS - default: false #### tikv.trust_cert_collection