diff --git a/CHANGELOG.md b/CHANGELOG.md index cd732110..476d8eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,16 +11,18 @@ All notable changes to this project will be documented in this file. a single namespace to watch ([#434]). - Support for ZooKeeper 3.8.0 added ([#464]). - Integration tests for all supported ZooKeeper versions added ([#464]). +- TLS encryption and authentication support for quorum and client ([#479]). ### Changed -- Operator-rs: 0.10.0 -> 0.15.0 ([#408], [#431], [#434], [#454]). +- Operator-rs: 0.10.0 -> 0.19.0 ([#408], [#431], [#434], [#454], [#479]). [#408]: https://github.com/stackabletech/zookeeper-operator/pull/408 [#431]: https://github.com/stackabletech/zookeeper-operator/pull/431 [#434]: https://github.com/stackabletech/zookeeper-operator/pull/434 [#454]: https://github.com/stackabletech/zookeeper-operator/pull/454 [#464]: https://github.com/stackabletech/zookeeper-operator/pull/464 +[#479]: https://github.com/stackabletech/zookeeper-operator/pull/479 ## [0.9.0] - 2022-02-14 diff --git a/Cargo.lock b/Cargo.lock index 6dfa5b74..a5f034be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,17 @@ version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" +[[package]] +name = "async-trait" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -133,6 +144,12 @@ dependencies = [ "git2", ] +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "byteorder" version = "1.4.3" @@ -204,9 +221,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.16" +version = "3.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52d4f8e4a1419219935762e32913b4430f37cb0c0200ad17a89ee18c0188a9f" +checksum = "85a35a599b11c089a7f49105658d089b8f2cf0882993c17daf6de15285c2c35d" dependencies = [ "atty", "bitflags", @@ -286,6 +303,16 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.8", +] + [[package]] name = "crossbeam-deque" version = "0.7.4" @@ -293,7 +320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" dependencies = [ "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "maybe-uninit", ] @@ -305,7 +332,7 @@ checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg", "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "lazy_static", "maybe-uninit", "memoffset", @@ -319,7 +346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" dependencies = [ "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "maybe-uninit", ] @@ -334,6 +361,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if 1.0.0", + "lazy_static", +] + [[package]] name = "darling" version = "0.13.4" @@ -781,6 +818,24 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-openssl" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ee5d7a8f718585d1c3c61dfde28ef5b0bb14734b4db13f5ada856cdc6c612b" +dependencies = [ + "http", + "hyper", + "linked_hash_set", + "once_cell", + "openssl", + "openssl-sys", + "parking_lot 0.12.0", + "tokio 1.18.1", + "tokio-openssl", + "tower-layer", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -842,6 +897,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "integer-encoding" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e85a1509a128c855368e135cffcde7eac17d8e1083f41e2b98c58bc1a5074be" + [[package]] name = "iovec" version = "0.1.4" @@ -877,6 +938,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "json-patch" version = "0.2.6" @@ -926,9 +996,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcc72fdf0c491160a34d4a1bfb03f96da8a5054288d61c816d514b5c2fa49ea" +checksum = "342744dfeb81fe186b84f485b33f12c6a15d3396987d933b06a566a3db52ca38" dependencies = [ "k8s-openapi", "kube-client", @@ -939,9 +1009,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b01c722d55ffedec74cbc259b4508d8a59bf19540006ec87618f76ab156579" +checksum = "3f69a504997799340408635d6e351afb8aab2c34ca3165e162f41b3b34a69a79" dependencies = [ "base64", "bytes 1.1.0", @@ -952,6 +1022,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-openssl", "hyper-timeout", "hyper-tls", "jsonpath_lib", @@ -975,9 +1046,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd9e3535777edd122cc26fe3fe6357066b33eff63d8b919862edbe7a956a679" +checksum = "a4a247487699941baaf93438d65b12d4e32450bea849d619d19ed394e8a4a645" dependencies = [ "chrono", "form_urlencoded", @@ -993,9 +1064,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1322e25c20dd6f18ca6baecc88bb130331e99d988df9d7a9a207f15819e05bff" +checksum = "203f7c5acf9d0dfb0b08d44ec1d66ace3d1dfe0cdd82e65e274f3f96615d666c" dependencies = [ "darling", "proc-macro2", @@ -1006,9 +1077,9 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816c8c086f8bbcf9a4db0b7a68db90b784ef6292a57de35c64cccb90d5edfbe5" +checksum = "02ea50e6ed56578e1d1d02548901b12fe6d3edbf110269a396955e285d487973" dependencies = [ "ahash", "backoff", @@ -1070,6 +1141,15 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "linked_hash_set" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lock_api" version = "0.3.4" @@ -1335,6 +1415,60 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand", + "thiserror", + "tokio 1.18.1", + "tokio-stream", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c0b12cd9e3f9b35b52f6e0dac66866c519b26f424f4bbf96e3fe8bfbdc5229" +dependencies = [ + "async-trait", + "lazy_static", + "opentelemetry", + "opentelemetry-semantic-conventions", + "thiserror", + "thrift", + "tokio 1.18.1", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "ordered-float" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "2.10.0" @@ -1484,17 +1618,17 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.38" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" dependencies = [ "unicode-xid", ] [[package]] name = "product-config" -version = "0.3.1" -source = "git+https://github.com/stackabletech/product-config.git?tag=0.3.1#40c93e5283beef100c9fecdb6368f1e1480db3e8" +version = "0.4.0" +source = "git+https://github.com/stackabletech/product-config.git?tag=0.4.0#e1e5938b4f6120f85a088194e86d22433fdba731" dependencies = [ "fancy-regex", "java-properties", @@ -1746,7 +1880,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" dependencies = [ - "ordered-float", + "ordered-float 2.10.0", "serde", ] @@ -1875,8 +2009,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.15.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.15.0#c7c408d476c0b7ba06833c19e5c9d2aefa5875ae" +version = "0.19.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.19.0#f8f2d5527b3463cc40f3d851172af7ae81db75c1" dependencies = [ "backoff", "chrono", @@ -1889,6 +2023,8 @@ dependencies = [ "k8s-openapi", "kube", "lazy_static", + "opentelemetry", + "opentelemetry-jaeger", "product-config", "rand", "regex", @@ -1896,13 +2032,26 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "stackable-operator-derive", "strum", "thiserror", "tokio 1.18.1", "tracing", + "tracing-opentelemetry", "tracing-subscriber", ] +[[package]] +name = "stackable-operator-derive" +version = "0.17.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.19.0#f8f2d5527b3463cc40f3d851172af7ae81db75c1" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "stackable-zookeeper-crd" version = "0.10.0-nightly" @@ -2049,6 +2198,28 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b82ca8f46f95b3ce96081fe3dd89160fdea970c254bb72925255d1b62aae692e" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float 1.1.1", + "threadpool", +] + [[package]] name = "time" version = "0.1.43" @@ -2145,7 +2316,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.31", ] @@ -2202,13 +2373,25 @@ dependencies = [ "tokio 1.18.1", ] +[[package]] +name = "tokio-openssl" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +dependencies = [ + "futures-util", + "openssl", + "openssl-sys", + "tokio 1.18.1", +] + [[package]] name = "tokio-reactor" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.31", "lazy_static", "log", @@ -2221,6 +2404,17 @@ dependencies = [ "tokio-sync", ] +[[package]] +name = "tokio-stream" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio 1.18.1", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -2253,7 +2447,7 @@ checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" dependencies = [ "crossbeam-deque", "crossbeam-queue", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.31", "lazy_static", "log", @@ -2268,7 +2462,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.31", "slab", "tokio-executor", @@ -2439,6 +2633,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f9378e96a9361190ae297e7f3a8ff644aacd2897f244b1ff81f381669196fa6" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-subscriber" version = "0.3.11" @@ -2545,6 +2752,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + [[package]] name = "winapi" version = "0.2.8" diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index aebefcde..20c23756 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -9,6 +9,7 @@ spec: - unit: &unitPort name: "port" regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" + properties: - property: &tickTime propertyNames: @@ -29,28 +30,6 @@ properties: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "The basic time unit in milliseconds used by ZooKeeper. It is used to do heartbeats and the minimum session timeout will be twice the tickTime." - - property: &clientPort - propertyNames: - - name: "clientPort" - kind: - type: "file" - file: "zoo.cfg" - datatype: - type: "integer" - unit: *unitPort - min: "1024" - max: "65535" - defaultValues: - - value: "2181" - recommendedValues: - - value: "2181" - roles: - - name: "server" - required: true - asOfVersion: "0.0.0" - comment: "See QuorumPeerConfig.java, I'm unsure what happens when this is set to 0, it might work, it might not" - description: "The port to listen for client connections; that is, the port that clients attempt to connect to." - - property: &dataDir propertyNames: - name: "dataDir" @@ -109,11 +88,34 @@ properties: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "Amount of time, in ticks (see `tickTime`), to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they will be dropped. In other words: The number of ticks that can pass between sending a request and getting an acknowledgment before a follower is dropped." - - property: &metricsPort + ########################## + # Ports + ########################## + - property: &clientPort + propertyNames: + - name: "clientPort" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "integer" + unit: *unitPort + min: "1024" + max: "65535" + defaultValues: + - value: "2181" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "The plaintext port to listen on for client connections." + + - property: &secureClientPort propertyNames: - - name: "metricsPort" + - name: "secureClientPort" kind: - type: "env" + type: "file" + file: "zoo.cfg" datatype: type: "integer" unit: *unitPort @@ -123,9 +125,9 @@ properties: - name: "server" required: false asOfVersion: "0.0.0" - description: "The port where ZooKeeper metrics are exposed as a Prometheus endpoint." + description: "The secure port to listen on for secure client connections using TLS/SSL." - - property: &admin_serverPort + - property: &adminServerPort propertyNames: - name: "admin.serverPort" kind: @@ -142,4 +144,258 @@ properties: - name: "server" required: true asOfVersion: "0.0.0" - description: "The zookeeper admin server port." \ No newline at end of file + description: "The port the embedded Jetty server listens on." + + ########################## + # Quorum TLS + ########################## + - property: &sslQuorum + propertyNames: + - name: "sslQuorum" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + defaultValues: + - value: "false" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Enables encrypted quorum communication." + + - property: &sslQuorumClientAuth + propertyNames: + - name: "ssl.quorum.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from other quorum members." + + - property: &sslQuorumHostNameVerification + propertyNames: + - name: "ssl.quorum.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in quorum TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslQuorumKeyStoreLocation + propertyNames: + - name: "ssl.quorum.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumKeyStorePassword + propertyNames: + - name: "ssl.quorum.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum keystore." + + - property: &sslQuorumTrustStoreLocation + propertyNames: + - name: "ssl.quorum.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumTrustStorePassword + propertyNames: + - name: "ssl.quorum.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum truststore." + + ########################## + # Client TLS + ########################## + - property: &sslClientAuth + propertyNames: + - name: "ssl.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from clients." + + - property: &sslHostNameVerification + propertyNames: + - name: "ssl.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in client TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslKeyStoreLocation + propertyNames: + - name: "ssl.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for client TLS connections." + + - property: &sslKeyStorePassword + propertyNames: + - name: "ssl.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client keystore." + + - property: &sslTrustStoreLocation + propertyNames: + - name: "ssl.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the remote credentials to be used for client TLS connections." + + - property: &sslTrustStorePassword + propertyNames: + - name: "ssl.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client truststore." + ########################## + # Common TLS + ########################## + - property: &authProviderX509 + propertyNames: + - name: "authProvider.x509" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "Used for secure client authentication." + + - property: &serverCnxnFactory + propertyNames: + - name: "serverCnxnFactory" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "This should be set to NettyServerCnxnFactory in order to use TLS based server communication." diff --git a/deploy/crd/zookeepercluster.crd.yaml b/deploy/crd/zookeepercluster.crd.yaml index f90256cd..a514ef7a 100644 --- a/deploy/crd/zookeepercluster.crd.yaml +++ b/deploy/crd/zookeepercluster.crd.yaml @@ -23,6 +23,31 @@ spec: spec: description: A cluster of ZooKeeper nodes properties: + config: + nullable: true + properties: + clientAuthentication: + description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" + nullable: true + properties: + authenticationClass: + type: string + required: + - authenticationClass + type: object + quorumTlsSecretClass: + description: "Only affects quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" + type: string + tls: + description: "Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: \"tls\".to_string() }." + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + type: object servers: nullable: true properties: diff --git a/deploy/helm/zookeeper-operator/configs/properties.yaml b/deploy/helm/zookeeper-operator/configs/properties.yaml index aebefcde..20c23756 100644 --- a/deploy/helm/zookeeper-operator/configs/properties.yaml +++ b/deploy/helm/zookeeper-operator/configs/properties.yaml @@ -9,6 +9,7 @@ spec: - unit: &unitPort name: "port" regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" + properties: - property: &tickTime propertyNames: @@ -29,28 +30,6 @@ properties: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "The basic time unit in milliseconds used by ZooKeeper. It is used to do heartbeats and the minimum session timeout will be twice the tickTime." - - property: &clientPort - propertyNames: - - name: "clientPort" - kind: - type: "file" - file: "zoo.cfg" - datatype: - type: "integer" - unit: *unitPort - min: "1024" - max: "65535" - defaultValues: - - value: "2181" - recommendedValues: - - value: "2181" - roles: - - name: "server" - required: true - asOfVersion: "0.0.0" - comment: "See QuorumPeerConfig.java, I'm unsure what happens when this is set to 0, it might work, it might not" - description: "The port to listen for client connections; that is, the port that clients attempt to connect to." - - property: &dataDir propertyNames: - name: "dataDir" @@ -109,11 +88,34 @@ properties: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "Amount of time, in ticks (see `tickTime`), to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they will be dropped. In other words: The number of ticks that can pass between sending a request and getting an acknowledgment before a follower is dropped." - - property: &metricsPort + ########################## + # Ports + ########################## + - property: &clientPort + propertyNames: + - name: "clientPort" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "integer" + unit: *unitPort + min: "1024" + max: "65535" + defaultValues: + - value: "2181" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "The plaintext port to listen on for client connections." + + - property: &secureClientPort propertyNames: - - name: "metricsPort" + - name: "secureClientPort" kind: - type: "env" + type: "file" + file: "zoo.cfg" datatype: type: "integer" unit: *unitPort @@ -123,9 +125,9 @@ properties: - name: "server" required: false asOfVersion: "0.0.0" - description: "The port where ZooKeeper metrics are exposed as a Prometheus endpoint." + description: "The secure port to listen on for secure client connections using TLS/SSL." - - property: &admin_serverPort + - property: &adminServerPort propertyNames: - name: "admin.serverPort" kind: @@ -142,4 +144,258 @@ properties: - name: "server" required: true asOfVersion: "0.0.0" - description: "The zookeeper admin server port." \ No newline at end of file + description: "The port the embedded Jetty server listens on." + + ########################## + # Quorum TLS + ########################## + - property: &sslQuorum + propertyNames: + - name: "sslQuorum" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + defaultValues: + - value: "false" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Enables encrypted quorum communication." + + - property: &sslQuorumClientAuth + propertyNames: + - name: "ssl.quorum.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from other quorum members." + + - property: &sslQuorumHostNameVerification + propertyNames: + - name: "ssl.quorum.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in quorum TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslQuorumKeyStoreLocation + propertyNames: + - name: "ssl.quorum.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumKeyStorePassword + propertyNames: + - name: "ssl.quorum.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum keystore." + + - property: &sslQuorumTrustStoreLocation + propertyNames: + - name: "ssl.quorum.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumTrustStorePassword + propertyNames: + - name: "ssl.quorum.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum truststore." + + ########################## + # Client TLS + ########################## + - property: &sslClientAuth + propertyNames: + - name: "ssl.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from clients." + + - property: &sslHostNameVerification + propertyNames: + - name: "ssl.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in client TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslKeyStoreLocation + propertyNames: + - name: "ssl.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for client TLS connections." + + - property: &sslKeyStorePassword + propertyNames: + - name: "ssl.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client keystore." + + - property: &sslTrustStoreLocation + propertyNames: + - name: "ssl.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the remote credentials to be used for client TLS connections." + + - property: &sslTrustStorePassword + propertyNames: + - name: "ssl.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client truststore." + ########################## + # Common TLS + ########################## + - property: &authProviderX509 + propertyNames: + - name: "authProvider.x509" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "Used for secure client authentication." + + - property: &serverCnxnFactory + propertyNames: + - name: "serverCnxnFactory" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "This should be set to NettyServerCnxnFactory in order to use TLS based server communication." diff --git a/deploy/helm/zookeeper-operator/crds/crds.yaml b/deploy/helm/zookeeper-operator/crds/crds.yaml index 5a59580a..3aa10904 100644 --- a/deploy/helm/zookeeper-operator/crds/crds.yaml +++ b/deploy/helm/zookeeper-operator/crds/crds.yaml @@ -25,6 +25,31 @@ spec: spec: description: A cluster of ZooKeeper nodes properties: + config: + nullable: true + properties: + clientAuthentication: + description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" + nullable: true + properties: + authenticationClass: + type: string + required: + - authenticationClass + type: object + quorumTlsSecretClass: + description: "Only affects quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" + type: string + tls: + description: "Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: \"tls\".to_string() }." + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + type: object servers: nullable: true properties: diff --git a/deploy/helm/zookeeper-operator/templates/roles.yaml b/deploy/helm/zookeeper-operator/templates/roles.yaml index cccdd90a..a4dca793 100644 --- a/deploy/helm/zookeeper-operator/templates/roles.yaml +++ b/deploy/helm/zookeeper-operator/templates/roles.yaml @@ -68,6 +68,14 @@ rules: - customresourcedefinitions verbs: - get + - apiGroups: + - authentication.stackable.tech + resources: + - authenticationclasses + verbs: + - get + - list + - watch - apiGroups: - events.k8s.io resources: diff --git a/deploy/manifests/configmap.yaml b/deploy/manifests/configmap.yaml index 281ebf16..2246ff29 100644 --- a/deploy/manifests/configmap.yaml +++ b/deploy/manifests/configmap.yaml @@ -1,7 +1,7 @@ --- apiVersion: v1 data: - properties.yaml: |- + properties.yaml: | version: 0.1.0 spec: units: @@ -13,6 +13,7 @@ data: - unit: &unitPort name: "port" regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" + properties: - property: &tickTime propertyNames: @@ -33,28 +34,6 @@ data: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "The basic time unit in milliseconds used by ZooKeeper. It is used to do heartbeats and the minimum session timeout will be twice the tickTime." - - property: &clientPort - propertyNames: - - name: "clientPort" - kind: - type: "file" - file: "zoo.cfg" - datatype: - type: "integer" - unit: *unitPort - min: "1024" - max: "65535" - defaultValues: - - value: "2181" - recommendedValues: - - value: "2181" - roles: - - name: "server" - required: true - asOfVersion: "0.0.0" - comment: "See QuorumPeerConfig.java, I'm unsure what happens when this is set to 0, it might work, it might not" - description: "The port to listen for client connections; that is, the port that clients attempt to connect to." - - property: &dataDir propertyNames: - name: "dataDir" @@ -113,11 +92,34 @@ data: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "Amount of time, in ticks (see `tickTime`), to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they will be dropped. In other words: The number of ticks that can pass between sending a request and getting an acknowledgment before a follower is dropped." - - property: &metricsPort + ########################## + # Ports + ########################## + - property: &clientPort + propertyNames: + - name: "clientPort" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "integer" + unit: *unitPort + min: "1024" + max: "65535" + defaultValues: + - value: "2181" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "The plaintext port to listen on for client connections." + + - property: &secureClientPort propertyNames: - - name: "metricsPort" + - name: "secureClientPort" kind: - type: "env" + type: "file" + file: "zoo.cfg" datatype: type: "integer" unit: *unitPort @@ -127,9 +129,9 @@ data: - name: "server" required: false asOfVersion: "0.0.0" - description: "The port where ZooKeeper metrics are exposed as a Prometheus endpoint." + description: "The secure port to listen on for secure client connections using TLS/SSL." - - property: &admin_serverPort + - property: &adminServerPort propertyNames: - name: "admin.serverPort" kind: @@ -146,7 +148,261 @@ data: - name: "server" required: true asOfVersion: "0.0.0" - description: "The zookeeper admin server port." + description: "The port the embedded Jetty server listens on." + + ########################## + # Quorum TLS + ########################## + - property: &sslQuorum + propertyNames: + - name: "sslQuorum" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + defaultValues: + - value: "false" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Enables encrypted quorum communication." + + - property: &sslQuorumClientAuth + propertyNames: + - name: "ssl.quorum.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from other quorum members." + + - property: &sslQuorumHostNameVerification + propertyNames: + - name: "ssl.quorum.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in quorum TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslQuorumKeyStoreLocation + propertyNames: + - name: "ssl.quorum.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumKeyStorePassword + propertyNames: + - name: "ssl.quorum.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum keystore." + + - property: &sslQuorumTrustStoreLocation + propertyNames: + - name: "ssl.quorum.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumTrustStorePassword + propertyNames: + - name: "ssl.quorum.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum truststore." + + ########################## + # Client TLS + ########################## + - property: &sslClientAuth + propertyNames: + - name: "ssl.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from clients." + + - property: &sslHostNameVerification + propertyNames: + - name: "ssl.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in client TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslKeyStoreLocation + propertyNames: + - name: "ssl.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for client TLS connections." + + - property: &sslKeyStorePassword + propertyNames: + - name: "ssl.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client keystore." + + - property: &sslTrustStoreLocation + propertyNames: + - name: "ssl.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the remote credentials to be used for client TLS connections." + + - property: &sslTrustStorePassword + propertyNames: + - name: "ssl.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client truststore." + ########################## + # Common TLS + ########################## + - property: &authProviderX509 + propertyNames: + - name: "authProvider.x509" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "Used for secure client authentication." + + - property: &serverCnxnFactory + propertyNames: + - name: "serverCnxnFactory" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "This should be set to NettyServerCnxnFactory in order to use TLS based server communication." kind: ConfigMap metadata: name: zookeeper-operator-configmap diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index f9dbdd71..b671142a 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -26,6 +26,31 @@ spec: spec: description: A cluster of ZooKeeper nodes properties: + config: + nullable: true + properties: + clientAuthentication: + description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" + nullable: true + properties: + authenticationClass: + type: string + required: + - authenticationClass + type: object + quorumTlsSecretClass: + description: "Only affects quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" + type: string + tls: + description: "Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: \"tls\".to_string() }." + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + type: object servers: nullable: true properties: diff --git a/deploy/manifests/roles.yaml b/deploy/manifests/roles.yaml index 8ff161da..616cda56 100644 --- a/deploy/manifests/roles.yaml +++ b/deploy/manifests/roles.yaml @@ -68,6 +68,14 @@ rules: - customresourcedefinitions verbs: - get + - apiGroups: + - authentication.stackable.tech + resources: + - authenticationclasses + verbs: + - get + - list + - watch - apiGroups: - events.k8s.io resources: diff --git a/docs/modules/ROOT/examples/example-cluster-tls-authentication-class.yaml b/docs/modules/ROOT/examples/example-cluster-tls-authentication-class.yaml new file mode 100644 index 00000000..1240b539 --- /dev/null +++ b/docs/modules/ROOT/examples/example-cluster-tls-authentication-class.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: zk-client-tls # <2> +spec: + provider: + tls: + clientCertSecretClass: zk-client-auth-secret # <3> diff --git a/docs/modules/ROOT/examples/example-cluster-tls-authentication-secret.yaml b/docs/modules/ROOT/examples/example-cluster-tls-authentication-secret.yaml new file mode 100644 index 00000000..e7bcd065 --- /dev/null +++ b/docs/modules/ROOT/examples/example-cluster-tls-authentication-secret.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: zk-client-auth-secret # <4> +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-zk-client-ca + namespace: default + autoGenerate: true diff --git a/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml b/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml new file mode 100644 index 00000000..45597957 --- /dev/null +++ b/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: simple-zk +spec: + version: 3.8.0 + config: + tls: + secretClass: tls + clientAuthentication: + authenticationClass: zk-client-tls # <1> + quorumTlsSecretClass: tls + servers: + roleGroups: + default: + replicas: 3 diff --git a/docs/modules/ROOT/examples/example-cluster-tls-encryption.yaml b/docs/modules/ROOT/examples/example-cluster-tls-encryption.yaml new file mode 100644 index 00000000..1f415f18 --- /dev/null +++ b/docs/modules/ROOT/examples/example-cluster-tls-encryption.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: simple-zk +spec: + version: 3.8.0 + config: + tls: + secretClass: tls # <1> + quorumTlsSecretClass: tls # <2> + servers: + roleGroups: + default: + replicas: 3 diff --git a/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml b/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml new file mode 100644 index 00000000..e6f27e9b --- /dev/null +++ b/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: tls +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-ca + namespace: default + autoGenerate: true diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 03c25635..b0e1d7c7 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -6,41 +6,79 @@ If you are not installing the operator using Helm then after installation the CR To create a three-node Apache ZooKeeper (v3.8.0) cluster: - - $ cat zookeeper.yaml - --- - apiVersion: zookeeper.stackable.tech/v1alpha1 - kind: ZookeeperCluster - metadata: - name: simple-zk - spec: - version: 3.8.0 - servers: - roleGroups: - default: - replicas: 3 - config: {} - $ kubectl apply -f zookeeper.yaml +[source,yaml] +---- +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: simple-zk +spec: + version: 3.8.0 + servers: + roleGroups: + default: + replicas: 3 + config: {} +---- Afterwards, a xref:znodes.adoc[ZNode] can be created: - $ cat znode.yaml - --- - apiVersion: zookeeper.stackable.tech/v1alpha1 - kind: ZookeeperZnode - metadata: - name: simple-znode - spec: - clusterRef: - name: simple-zk - namespace: default - $ kubectl apply -f znode.yaml +[source,yaml] +---- +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: simple-znode +spec: + clusterRef: + name: simple-zk + namespace: default +---- Finally, a ConfigMap is created, containing a path that a ZooKeeper client can connect to: $ kubectl get configmap simple-znode-nodeport -o yaml $ $ZOOKEEPER_HOME/bin/zkCli.sh -server $(kubectl get configmap simple-znode-nodeport -o jsonpath='{.data.ZOOKEEPER}') - + +== Encryption + +The quorum and client communication are encrypted by default via TLS. This requires the https://github.com/stackabletech/secret-operator[Secret Operator] to be present in order to provide certificates. The utilized certificates can be changed in a top-level config. + +[source,yaml] +---- +include::example$example-cluster-tls-encryption.yaml[] +---- +<1> The `tls.secretClass` refers to the client-to-server encryption. Defaults to the `tls` secret. +<2> The `quorumTlsSecretClass` refers to the server-to-server quorum encryption. Defaults to the `tls` secret. + +The `tls` secret is deployed from the https://github.com/stackabletech/secret-operator[Secret Operator] and looks like this: + +[source,yaml] +---- +include::example$example-secret-operator-tls-secret.yaml[] +---- + +You can create your own secrets and reference them e.g. in the `tls.secretClass` to use different certificates. + +== Authentication + +The quorum or server-to-server communication is authenticated via TLS per default. In order to enforce TLS authentication for client-to-server communication, you can set an `AuthenticationClass` reference in the custom resource provided by the https://github.com/stackabletech/commons-operator[Commons Operator]. + +[source,yaml] +---- +include::example$example-cluster-tls-authentication.yaml[] +include::example$example-cluster-tls-authentication-class.yaml[] +include::example$example-cluster-tls-authentication-secret.yaml[] +---- +<1> The `config.clientAuthentication.authenticationClass` can be set to use TLS for authentication. This is optional. +<2> The referenced `AuthenticationClass` that references a `SecretClass` to provide certificates. +<3> The reference to a `SecretClass`. +<4> The `SecretClass` that is referenced by the `AuthenticationClass` in order to provide certificates. + +WARNING: Due to a https://issues.apache.org/jira/browse/ZOOKEEPER-4276[bug] in ZooKeeper, the `clientPort` property in combination with `client.portUnification=true` is used instead of the `secureClientPort`. This means that unencrypted and unauthenticated access to the ZooKeeper cluster is still possible. + == Monitoring The managed ZooKeeper instances are automatically configured to export Prometheus metrics. See diff --git a/examples/simple-zookeeper-tls-cluster.yaml b/examples/simple-zookeeper-tls-cluster.yaml new file mode 100644 index 00000000..e8285656 --- /dev/null +++ b/examples/simple-zookeeper-tls-cluster.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: simple-zk +spec: + version: 3.8.0 + config: + tls: + secretClass: tls + clientAuthentication: + authenticationClass: zk-client-tls + quorumTlsSecretClass: tls + servers: + roleGroups: + default: + replicas: 3 +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: zk-client-tls +spec: + provider: + tls: + clientCertSecretClass: zk-client-auth-secret +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: zk-client-auth-secret +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-zk-client-ca + namespace: default + autoGenerate: true +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: simple-znode +spec: + clusterRef: + name: simple-zk + # Optional when ZookeeperZnode is in the same Namespace as the ZookeeperCluster + # namespace: default diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index f3298c47..8ac4091c 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -11,5 +11,5 @@ repository = "https://github.com/stackabletech/zookeeper-operator" serde = "1.0.136" serde_json = "1.0.79" snafu = "0.7.0" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.19.0" } strum = { version = "0.24.0", features = ["derive"] } diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 40e557dc..ad08839e 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use snafu::{OptionExt, Snafu}; use stackable_operator::{ @@ -9,6 +7,19 @@ use stackable_operator::{ role_utils::{Role, RoleGroupRef}, schemars::{self, JsonSchema}, }; +use std::collections::BTreeMap; + +pub const CLIENT_PORT: u16 = 2181; +pub const SECURE_CLIENT_PORT: u16 = 2282; + +pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; +pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; +pub const STACKABLE_RW_CONFIG_DIR: &str = "/stackable/rwconfig"; + +pub const QUORUM_TLS_DIR: &str = "/stackable/tls/quorum"; +pub const CLIENT_TLS_DIR: &str = "/stackable/tls/client"; + +const DEFAULT_SECRET_CLASS: &str = "tls"; /// A cluster of ZooKeeper nodes #[derive(Clone, CustomResource, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -36,6 +47,67 @@ pub struct ZookeeperClusterSpec { pub version: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub servers: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config: Option, +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GlobalZookeeperConfig { + #[serde(flatten)] + pub tls_config: Option, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TlsConfig { + /// Only affects client connections. This setting controls: + /// - If TLS encryption is used at all + /// - Which cert the servers should use to authenticate themselves against the client + /// Defaults to `TlsSecretClass` { secret_class: "tls".to_string() }. + pub tls: Option, + /// Only affects client connections. This setting controls: + /// - If clients need to authenticate themselves against the server via TLS + /// - Which ca.crt to use when validating the provided client certs + /// Defaults to `None` + pub client_authentication: Option, + /// Only affects quorum communication. Use mutual verification between Zookeeper Nodes + /// (mandatory). This setting controls: + /// - Which cert the servers should use to authenticate themselves against other servers + /// - Which ca.crt to use when validating the other server + pub quorum_tls_secret_class: String, +} + +impl Default for TlsConfig { + fn default() -> Self { + Self { + tls: Some(TlsSecretClass { + secret_class: DEFAULT_SECRET_CLASS.to_string(), + }), + client_authentication: None, + quorum_tls_secret_class: DEFAULT_SECRET_CLASS.to_string(), + } + } +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TlsSecretClass { + pub secret_class: String, +} + +impl Default for TlsSecretClass { + fn default() -> Self { + Self { + secret_class: DEFAULT_SECRET_CLASS.to_string(), + } + } +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ClientAuthenticationClass { + pub authentication_class: String, } #[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -51,9 +123,31 @@ impl ZookeeperConfig { pub const INIT_LIMIT: &'static str = "initLimit"; pub const SYNC_LIMIT: &'static str = "syncLimit"; pub const TICK_TIME: &'static str = "tickTime"; + pub const DATA_DIR: &'static str = "dataDir"; pub const MYID_OFFSET: &'static str = "MYID_OFFSET"; pub const SERVER_JVMFLAGS: &'static str = "SERVER_JVMFLAGS"; + pub const CLIENT_PORT: &'static str = "clientPort"; + + // Quorum TLS + pub const SSL_QUORUM: &'static str = "sslQuorum"; + pub const SSL_QUORUM_CLIENT_AUTH: &'static str = "ssl.quorum.clientAuth"; + pub const SSL_QUORUM_HOST_NAME_VERIFICATION: &'static str = "ssl.quorum.hostnameVerification"; + pub const SSL_QUORUM_KEY_STORE_LOCATION: &'static str = "ssl.quorum.keyStore.location"; + pub const SSL_QUORUM_KEY_STORE_PASSWORD: &'static str = "ssl.quorum.keyStore.password"; + pub const SSL_QUORUM_TRUST_STORE_LOCATION: &'static str = "ssl.quorum.trustStore.location"; + pub const SSL_QUORUM_TRUST_STORE_PASSWORD: &'static str = "ssl.quorum.trustStore.password"; + // Client TLS + pub const SECURE_CLIENT_PORT: &'static str = "secureClientPort"; + pub const SSL_CLIENT_AUTH: &'static str = "ssl.clientAuth"; + pub const SSL_HOST_NAME_VERIFICATION: &'static str = "ssl.hostnameVerification"; + pub const SSL_KEY_STORE_LOCATION: &'static str = "ssl.keyStore.location"; + pub const SSL_KEY_STORE_PASSWORD: &'static str = "ssl.keyStore.password"; + pub const SSL_TRUST_STORE_LOCATION: &'static str = "ssl.trustStore.location"; + pub const SSL_TRUST_STORE_PASSWORD: &'static str = "ssl.trustStore.password"; + // Common TLS + pub const SSL_AUTH_PROVIDER_X509: &'static str = "authProvider.x509"; + pub const SERVER_CNXN_FACTORY: &'static str = "serverCnxnFactory"; fn myid_offset(&self) -> u16 { self.myid_offset.unwrap_or(1) @@ -89,7 +183,7 @@ impl Configuration for ZookeeperConfig { fn compute_files( &self, - _resource: &Self::Configurable, + resource: &Self::Configurable, _role_name: &str, _file: &str, ) -> Result>, ConfigError> { @@ -103,6 +197,106 @@ impl Configuration for ZookeeperConfig { if let Some(tick_time) = self.tick_time { result.insert(Self::TICK_TIME.to_string(), Some(tick_time.to_string())); } + result.insert( + Self::DATA_DIR.to_string(), + Some(STACKABLE_DATA_DIR.to_string()), + ); + + // Quorum TLS + result.insert(Self::SSL_QUORUM.to_string(), Some("true".to_string())); + result.insert( + Self::SSL_QUORUM_HOST_NAME_VERIFICATION.to_string(), + Some("true".to_string()), + ); + result.insert( + Self::SSL_QUORUM_CLIENT_AUTH.to_string(), + Some("need".to_string()), + ); + result.insert( + Self::SERVER_CNXN_FACTORY.to_string(), + Some("org.apache.zookeeper.server.NettyServerCnxnFactory".to_string()), + ); + result.insert( + Self::SSL_AUTH_PROVIDER_X509.to_string(), + Some("org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string()), + ); + // The keystore and truststore passwords should not be in the configmap and are generated + // and written later via script in the init container + result.insert( + Self::SSL_QUORUM_KEY_STORE_LOCATION.to_string(), + Some(format!("{dir}/keystore.p12", dir = QUORUM_TLS_DIR)), + ); + result.insert( + Self::SSL_QUORUM_TRUST_STORE_LOCATION.to_string(), + Some(format!("{dir}/truststore.p12", dir = QUORUM_TLS_DIR)), + ); + + // Client TLS + if resource.is_client_secure() { + // We set only the clientPort and portUnification here because otherwise there is a port bind exception + // See: https://issues.apache.org/jira/browse/ZOOKEEPER-4276 + // --> Normally we would like to only set the secureClientPort (check out commented code below) + // What we tried: + // 1) Set clientPort and secureClientPort will fail with + // "static.config different from dynamic config .. " + // result.insert( + // Self::CLIENT_PORT.to_string(), + // Some(CLIENT_PORT.to_string()), + // ); + // result.insert( + // Self::SECURE_CLIENT_PORT.to_string(), + // Some(SECURE_CLIENT_PORT.to_string()), + // ); + + // 2) Setting only secureClientPort will result in the above mentioned bind exception. + // The NettyFactory tries to bind multiple times on the secureClientPort. + // result.insert( + // Self::SECURE_CLIENT_PORT.to_string(), + // Some(resource.client_port().to_string()), + // ); + + // 3) Using the clientPort and portUnification still allows plaintext connection without + // authentication, but at least TLS and authentication works when connecting securely. + result.insert( + Self::CLIENT_PORT.to_string(), + Some(resource.client_port().to_string()), + ); + result.insert( + "client.portUnification".to_string(), + Some("true".to_string()), + ); + // TODO: Remove clientPort and portUnification in favor of secureClientPort once the bug is fixed + // result.insert( + // Self::SECURE_CLIENT_PORT.to_string(), + // Some(resource.client_port().to_string()), + // ); + // END TICKET + + result.insert( + Self::SSL_HOST_NAME_VERIFICATION.to_string(), + Some("true".to_string()), + ); + // The keystore and truststore passwords should not be in the configmap and are generated + // and written later via script in the init container + result.insert( + Self::SSL_KEY_STORE_LOCATION.to_string(), + Some(format!("{dir}/keystore.p12", dir = CLIENT_TLS_DIR)), + ); + result.insert( + Self::SSL_TRUST_STORE_LOCATION.to_string(), + Some(format!("{dir}/truststore.p12", dir = CLIENT_TLS_DIR)), + ); + // Check if we need to enable authentication + if resource.client_tls_authentication_class().is_some() { + result.insert(Self::SSL_CLIENT_AUTH.to_string(), Some("need".to_string())); + } + } else { + result.insert( + Self::CLIENT_PORT.to_string(), + Some(resource.client_port().to_string()), + ); + } + Ok(result) } } @@ -179,6 +373,56 @@ impl ZookeeperCluster { }) })) } + + pub fn client_port(&self) -> u16 { + if self.is_client_secure() { + SECURE_CLIENT_PORT + } else { + CLIENT_PORT + } + } + + /// Returns the secret class for client connection encryption. Defaults to `tls`. + pub fn client_tls_secret_class(&self) -> String { + let spec: &ZookeeperClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.tls_config.as_ref()) + .and_then(|tls| tls.tls.clone()) + .unwrap_or_default() + .secret_class + } + + /// Checks if we should use TLS to encrypt client connections. + pub fn is_client_secure(&self) -> bool { + let spec: &ZookeeperClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.tls_config.as_ref()) + .and_then(|tls| tls.tls.as_ref()) + .is_some() + } + + /// Returns the authentication class used for client authentication + pub fn client_tls_authentication_class(&self) -> Option { + let spec: &ZookeeperClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.tls_config.as_ref()) + .and_then(|tls| tls.client_authentication.as_ref()) + .map(|auth| auth.authentication_class.clone()) + } + + /// Returns the secret class for internal server encryption + pub fn quorum_tls_secret_class(&self) -> String { + let spec: &ZookeeperClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.tls_config.as_ref()) + .map(|tls| tls.quorum_tls_secret_class.as_ref()) + .unwrap_or(DEFAULT_SECRET_CLASS) + .to_string() + } } /// Reference to a single `Pod` that is a component of a [`ZookeeperCluster`] diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 49f90d67..7caea61a 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -25,10 +25,10 @@ tokio-executor = "0.1.10" tokio-zookeeper = "0.1.3" tracing = "0.1.34" pin-project = "1.0.10" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.19.0" } stackable-zookeeper-crd = { path = "../crd" } [build-dependencies] built = { version = "0.5.1", features = ["chrono", "git2"] } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.19.0" } stackable-zookeeper-crd = { path = "../crd" } diff --git a/rust/operator-binary/src/command.rs b/rust/operator-binary/src/command.rs new file mode 100644 index 00000000..7187ba0a --- /dev/null +++ b/rust/operator-binary/src/command.rs @@ -0,0 +1,102 @@ +use stackable_zookeeper_crd::{ + ZookeeperCluster, ZookeeperConfig, CLIENT_TLS_DIR, QUORUM_TLS_DIR, STACKABLE_CONFIG_DIR, + STACKABLE_DATA_DIR, STACKABLE_RW_CONFIG_DIR, +}; + +const STORE_PASSWORD_ENV: &str = "STORE_PASSWORD"; + +pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { + let mut args = vec![]; + + // copy config files to a writeable empty folder in order to set key and + // truststore passwords in the initcontainer via script + args.extend(vec![ + format!( + "echo copying {conf} to {rw_conf}", + conf = STACKABLE_CONFIG_DIR, + rw_conf = STACKABLE_RW_CONFIG_DIR + ), + format!( + "cp -RL {conf}/* {rw_conf}", + conf = STACKABLE_CONFIG_DIR, + rw_conf = STACKABLE_RW_CONFIG_DIR + ), + ]); + + // Quorum + args.push(generate_password()); + args.extend(create_key_and_trust_store_cmd(QUORUM_TLS_DIR)); + args.extend(vec![ + write_store_password_to_config(ZookeeperConfig::SSL_QUORUM_KEY_STORE_PASSWORD), + write_store_password_to_config(ZookeeperConfig::SSL_QUORUM_TRUST_STORE_PASSWORD), + ]); + + // Client + if zk.is_client_secure() { + args.push(generate_password()); + args.extend(create_key_and_trust_store_cmd(CLIENT_TLS_DIR)); + + args.extend(vec![ + write_store_password_to_config(ZookeeperConfig::SSL_KEY_STORE_PASSWORD), + write_store_password_to_config(ZookeeperConfig::SSL_TRUST_STORE_PASSWORD), + ]); + } + + args.extend([ + format!("chown stackable:stackable {dir}", dir = STACKABLE_DATA_DIR), + format!("chmod a=,u=rwX {dir}", dir = STACKABLE_DATA_DIR), + format!( + "chown -R stackable:stackable {rwconf_directory}", + rwconf_directory = STACKABLE_RW_CONFIG_DIR + ), + format!( + "chmod -R a=,u=rwX {rwconf_directory}", + rwconf_directory = STACKABLE_RW_CONFIG_DIR + ), + format!( + "expr $MYID_OFFSET + $(echo $POD_NAME | sed 's/.*-//') > {dir}/myid", + dir = STACKABLE_DATA_DIR + ), + ]); + + args.join(" && ") +} + +/// Generates the shell script to retrieve a random 20 character password +fn generate_password() -> String { + format!("export {STORE_PASSWORD_ENV}=$(tr -dc A-Za-z0-9 String { + format!( + "echo {property}=${STORE_PASSWORD_ENV} >> {rwconf}/zoo.cfg", + property = property, + rwconf = STACKABLE_RW_CONFIG_DIR + ) +} + +/// Generates the shell script to create key and truststores from the certificates provided +/// by the secret operator +fn create_key_and_trust_store_cmd(directory: &str) -> Vec { + vec![ + format!("echo [{dir}] Storing password", dir = directory), + format!("echo ${STORE_PASSWORD_ENV} > {dir}/password", dir = directory), + format!("echo [{dir}] Cleaning up truststore - just in case", dir = directory), + format!("rm -f {dir}/truststore.p12", dir = directory), + format!("echo [{dir}] Creating truststore", dir = directory), + format!("keytool -importcert -file {dir}/ca.crt -keystore {dir}/truststore.p12 -storetype pkcs12 -noprompt -alias ca_cert -storepass ${STORE_PASSWORD_ENV}", dir = directory), + format!("echo [{dir}] Creating certificate chain", dir = directory), + format!("cat {dir}/ca.crt {dir}/tls.crt > {dir}/chain.crt", dir = directory), + format!("echo [{dir}] Creating keystore", dir = directory), + format!("openssl pkcs12 -export -in {dir}/chain.crt -inkey {dir}/tls.key -out {dir}/keystore.p12 --passout file:{dir}/password", + dir = directory), + format!("echo [{dir}] Cleaning up password", dir = directory), + format!("rm -f {dir}/password", dir = directory), + format!("echo [{dir}] Chowning store directory", dir = directory), + format!("chown -R stackable:stackable {dir}", dir = directory), + format!("echo [{dir}] Chmodding store directory", dir = directory), + format!("chmod -R a=,u=rwX {dir}", dir = directory), + ] +} diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index f0a0ab6b..4af2d659 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -8,7 +8,7 @@ use stackable_operator::{ }; use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperRole}; -use crate::{zk_controller::zk_version, APP_NAME, APP_PORT}; +use crate::{zk_controller::zk_version, APP_NAME}; #[derive(Snafu, Debug)] pub enum Error { @@ -122,7 +122,7 @@ fn pod_hosts(zk: &ZookeeperCluster) -> Result anyhow::Result<()> { - stackable_operator::logging::initialize_logging("ZOOKEEPER_OPERATOR_LOG"); // tokio-zookeeper depends on Tokio 0.1 let tokio01_runtime = tokio01::runtime::Runtime::new()?; @@ -51,7 +50,13 @@ async fn main() -> anyhow::Result<()> { Command::Run(ProductOperatorRun { product_config, watch_namespace, + tracing_target, }) => { + stackable_operator::logging::initialize_logging( + "ZOOKEEPER_OPERATOR_LOG", + APP_NAME, + tracing_target, + ); stackable_operator::utils::print_startup_string( built_info::PKG_DESCRIPTION, built_info::PKG_VERSION, diff --git a/rust/operator-binary/src/zk_controller.rs b/rust/operator-binary/src/zk_controller.rs index 4dec5e81..816d1281 100644 --- a/rust/operator-binary/src/zk_controller.rs +++ b/rust/operator-binary/src/zk_controller.rs @@ -9,19 +9,24 @@ use std::{ }; use crate::{ + command::create_init_container_command_args, discovery::{self, build_discovery_configmaps}, - APP_NAME, APP_PORT, + ObjectRef, APP_NAME, }; use fnv::FnvHasher; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ - builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, + builder::{ + ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, + SecretOperatorVolumeSourceBuilder, VolumeBuilder, + }, + commons::authentication::{AuthenticationClass, AuthenticationClassProvider}, k8s_openapi::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, ConfigMapVolumeSource, EnvVar, EnvVarSource, ExecAction, - ObjectFieldSelector, PersistentVolumeClaim, PersistentVolumeClaimSpec, + ConfigMap, ConfigMapVolumeSource, EmptyDirVolumeSource, EnvVar, EnvVarSource, + ExecAction, ObjectFieldSelector, PersistentVolumeClaim, PersistentVolumeClaimSpec, PodSecurityContext, Probe, ResourceRequirements, SecurityContext, Service, ServicePort, ServiceSpec, Volume, }, @@ -29,7 +34,7 @@ use stackable_operator::{ apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, }, kube::{ - api::ObjectMeta, + api::{DynamicObject, ObjectMeta}, runtime::controller::{self, Context}, }, labels::{role_group_selector_labels, role_selector_labels}, @@ -40,7 +45,10 @@ use stackable_operator::{ product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::RoleGroupRef, }; -use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperClusterStatus, ZookeeperRole}; +use stackable_zookeeper_crd::{ + ZookeeperCluster, ZookeeperClusterStatus, ZookeeperRole, CLIENT_TLS_DIR, QUORUM_TLS_DIR, + STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_RW_CONFIG_DIR, +}; use strum::{EnumDiscriminants, IntoStaticStr}; const FIELD_MANAGER_SCOPE: &str = "zookeepercluster"; @@ -117,6 +125,21 @@ pub enum Error { ApplyStatus { source: stackable_operator::error::Error, }, + #[snafu(display("failed to retrieve {}", authentication_class))] + AuthenticationClassRetrieval { + source: stackable_operator::error::Error, + authentication_class: ObjectRef, + }, + #[snafu(display( + "failed to use authentication mechanism {} - supported methods: {:?}", + method, + supported + ))] + AuthenticationMethodNotSupported { + authentication_class: ObjectRef, + supported: Vec, + method: String, + }, } type Result = std::result::Result; @@ -124,6 +147,35 @@ impl ReconcilerError for Error { fn category(&self) -> &'static str { ErrorDiscriminants::from(self).into() } + fn secondary_object(&self) -> Option> { + match self { + Error::ObjectHasNoNamespace => None, + Error::ObjectHasNoVersion => None, + Error::NoServerRole => None, + Error::GlobalServiceNameNotFound => None, + Error::RoleGroupServiceNameNotFound { .. } => None, + Error::ApplyRoleService { .. } => None, + Error::ApplyRoleGroupService { .. } => None, + Error::BuildRoleGroupConfig { .. } => None, + Error::ApplyRoleGroupConfig { .. } => None, + Error::ApplyRoleGroupStatefulSet { .. } => None, + Error::GenerateProductConfig { .. } => None, + Error::InvalidProductConfig { .. } => None, + Error::SerializeZooCfg { .. } => None, + Error::ObjectMissingMetadataForOwnerRef { .. } => None, + Error::BuildDiscoveryConfig { .. } => None, + Error::ApplyDiscoveryConfig { .. } => None, + Error::ApplyStatus { .. } => None, + Error::AuthenticationClassRetrieval { + authentication_class, + .. + } => Some(authentication_class.clone().erase()), + Error::AuthenticationMethodNotSupported { + authentication_class, + .. + } => Some(authentication_class.clone().erase()), + } + } } const PROPERTIES_FILE: &str = "zoo.cfg"; @@ -157,11 +209,26 @@ pub async fn reconcile_zk( false, ) .context(InvalidProductConfigSnafu)?; + let role_server_config = validated_config .get(&ZookeeperRole::Server.to_string()) .map(Cow::Borrowed) .unwrap_or_default(); + let client_authentication_class = if let Some(auth_class) = zk.client_tls_authentication_class() + { + Some( + client + .get::(&auth_class, None) // AuthenticationClass has ClusterScope + .await + .context(AuthenticationClassRetrievalSnafu { + authentication_class: ObjectRef::::new(&auth_class), + })?, + ) + } else { + None + }; + let server_role_service = build_server_role_service(&zk)?; let server_role_service = client .apply_patch( @@ -176,7 +243,12 @@ pub async fn reconcile_zk( let rg_service = build_server_rolegroup_service(&rolegroup, &zk)?; let rg_configmap = build_server_rolegroup_config_map(&rolegroup, &zk, rolegroup_config)?; - let rg_statefulset = build_server_rolegroup_statefulset(&rolegroup, &zk, rolegroup_config)?; + let rg_statefulset = build_server_rolegroup_statefulset( + &rolegroup, + &zk, + rolegroup_config, + client_authentication_class.as_ref(), + )?; client .apply_patch(FIELD_MANAGER_SCOPE, &rg_service, &rg_service) .await @@ -247,7 +319,7 @@ pub fn build_server_role_service(zk: &ZookeeperCluster) -> Result { spec: Some(ServiceSpec { ports: Some(vec![ServicePort { name: Some("zk".to_string()), - port: APP_PORT.into(), + port: zk.client_port().into(), protocol: Some("TCP".to_string()), ..ServicePort::default() }]), @@ -272,9 +344,12 @@ fn build_server_rolegroup_config_map( zoo_cfg.extend(zk.pods().into_iter().flatten().map(|pod| { ( format!("server.{}", pod.zookeeper_myid), - format!("{}:2888:3888;{}", pod.fqdn(), APP_PORT), + format!("{}:2888:3888;{}", pod.fqdn(), zk.client_port()), ) })); + + // TLS + let zoo_cfg = zoo_cfg .into_iter() .map(|(k, v)| (k, Some(v))) @@ -336,7 +411,7 @@ fn build_server_rolegroup_service( ports: Some(vec![ ServicePort { name: Some("zk".to_string()), - port: APP_PORT.into(), + port: zk.client_port().into(), protocol: Some("TCP".to_string()), ..ServicePort::default() }, @@ -367,6 +442,7 @@ fn build_server_rolegroup_statefulset( rolegroup_ref: &RoleGroupRef, zk: &ZookeeperCluster, server_config: &HashMap>, + client_authentication_class: Option<&AuthenticationClass>, ) -> Result { let rolegroup = zk .spec @@ -390,18 +466,12 @@ fn build_server_rolegroup_statefulset( ..EnvVar::default() }) .collect::>(); - let mut container_prepare = ContainerBuilder::new("prepare") + + let mut container_prepare = ContainerBuilder::new("prepare"); + container_prepare .image(&image) - .args(vec![ - "sh".to_string(), - "-c".to_string(), - [ - "chown stackable:stackable /stackable/data", - "chmod a=,u=rwX /stackable/data", - "expr $MYID_OFFSET + $(echo $POD_NAME | sed 's/.*-//') > /stackable/data/myid", - ] - .join(" && "), - ]) + .command(vec!["sh".to_string(), "-c".to_string()]) + .args(vec![create_init_container_command_args(zk)]) .add_env_vars(env.clone()) .add_env_vars(vec![EnvVar { name: "POD_NAME".to_string(), @@ -414,18 +484,28 @@ fn build_server_rolegroup_statefulset( }), ..EnvVar::default() }]) - .add_volume_mount("data", "/stackable/data") - .build(); + .add_volume_mount("data", STACKABLE_DATA_DIR) + .add_volume_mount("quorum-tls", QUORUM_TLS_DIR) + .add_volume_mount("config", STACKABLE_CONFIG_DIR) + .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR); + + if zk.is_client_secure() { + container_prepare.add_volume_mount("client-tls", CLIENT_TLS_DIR); + } + + let mut container_prepare = container_prepare.build(); container_prepare .security_context .get_or_insert_with(SecurityContext::default) .run_as_user = Some(0); - let container_zk = ContainerBuilder::new("zookeeper") + + let mut container_zk = ContainerBuilder::new("zookeeper"); + container_zk .image(image) .args(vec![ "bin/zkServer.sh".to_string(), "start-foreground".to_string(), - "/stackable/config/zoo.cfg".to_string(), + format!("{dir}/zoo.cfg", dir = STACKABLE_RW_CONFIG_DIR), ]) .add_env_vars(env) // Only allow the global load balancing service to send traffic to pods that are members of the quorum @@ -439,20 +519,104 @@ fn build_server_rolegroup_statefulset( // we can use Bash's virtual /dev/tcp filesystem to accomplish the same thing format!( "exec 3<>/dev/tcp/localhost/{} && echo srvr >&3 && grep '^Mode: ' <&3", - APP_PORT + zk.client_port() ), ]), }), period_seconds: Some(1), ..Probe::default() }) - .add_container_port("zk", APP_PORT.into()) + .add_container_port("zk", zk.client_port().into()) .add_container_port("zk-leader", 2888) .add_container_port("zk-election", 3888) .add_container_port("metrics", 9505) - .add_volume_mount("data", "/stackable/data") - .add_volume_mount("config", "/stackable/config") - .build(); + .add_volume_mount("data", STACKABLE_DATA_DIR) + .add_volume_mount("config", STACKABLE_CONFIG_DIR) + .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR) + .add_volume_mount("quorum-tls", QUORUM_TLS_DIR); + + if zk.is_client_secure() { + container_zk.add_volume_mount("client-tls", CLIENT_TLS_DIR); + } + + let container_zk = container_zk.build(); + + let mut pod_builder = PodBuilder::new(); + pod_builder + .metadata_builder(|m| { + m.with_recommended_labels( + zk, + APP_NAME, + zk_version, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + }) + .add_init_container(container_prepare) + .add_container(container_zk) + .add_volume(Volume { + name: "config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: Some(rolegroup_ref.object_name()), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .add_volume(Volume { + empty_dir: Some(EmptyDirVolumeSource { + medium: None, + size_limit: None, + }), + name: "rwconfig".to_string(), + ..Volume::default() + }) + .add_volume( + VolumeBuilder::new("quorum-tls") + .csi( + SecretOperatorVolumeSourceBuilder::new(zk.quorum_tls_secret_class()) + .with_node_scope() + .with_pod_scope() + .build(), + ) + .build(), + ) + .security_context(PodSecurityContext { + fs_group: Some(1000), + ..PodSecurityContext::default() + }); + + if zk.is_client_secure() { + let secret_class = if let Some(auth_class) = client_authentication_class { + match &auth_class.spec.provider { + AuthenticationClassProvider::Tls(tls) => tls.client_cert_secret_class.clone(), + _ => { + return Err(Error::AuthenticationMethodNotSupported { + authentication_class: ObjectRef::from_obj(auth_class), + supported: vec!["tls".to_string()], + method: auth_class.spec.provider.to_string(), + }) + } + } + } else { + None + }; + + pod_builder.add_volume( + VolumeBuilder::new("client-tls") + .csi( + SecretOperatorVolumeSourceBuilder::new( + secret_class.unwrap_or_else(|| zk.client_tls_secret_class()), + ) + .with_node_scope() + .with_pod_scope() + .build(), + ) + .build(), + ); + } + + let pod_builder = pod_builder.build_template(); + Ok(StatefulSet { metadata: ObjectMetaBuilder::new() .name_and_namespace(zk) @@ -484,31 +648,7 @@ fn build_server_rolegroup_statefulset( ..LabelSelector::default() }, service_name: rolegroup_ref.object_name(), - template: PodBuilder::new() - .metadata_builder(|m| { - m.with_recommended_labels( - zk, - APP_NAME, - zk_version, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - }) - .add_init_container(container_prepare) - .add_container(container_zk) - .add_volume(Volume { - name: "config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: Some(rolegroup_ref.object_name()), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .security_context(PodSecurityContext { - fs_group: Some(1000), - ..PodSecurityContext::default() - }) - .build_template(), + template: pod_builder, volume_claim_templates: Some(vec![PersistentVolumeClaim { metadata: ObjectMeta { name: Some("data".to_string()), diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index 753fd5d6..811053f3 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -4,10 +4,7 @@ use std::{convert::Infallible, sync::Arc, time::Duration}; -use crate::{ - discovery::{self, build_discovery_configmaps}, - APP_PORT, -}; +use crate::discovery::{self, build_discovery_configmaps}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ k8s_openapi::api::core::v1::{ConfigMap, Service}, @@ -244,7 +241,7 @@ fn zk_mgmt_addr(zk: &ZookeeperCluster) -> Result { .with_context(|| NoZkFqdnSnafu { zk: ObjectRef::from_obj(zk), })?, - APP_PORT, + zk.client_port(), )) } diff --git a/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 b/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 index 633d42b2..36718c17 100644 --- a/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 @@ -4,6 +4,12 @@ kind: ZookeeperCluster metadata: name: test-zk spec: + config: + tls: + secretClass: tls + clientAuthentication: + authenticationClass: zk-client-tls + quorumTlsSecretClass: tls servers: roleGroups: primary: @@ -17,6 +23,28 @@ spec: version: {{ test_scenario['values']['zookeeper'] }} stopped: false --- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: zk-client-tls +spec: + provider: + tls: + clientCertSecretClass: zk-client-auth-secret +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: zk-client-auth-secret +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-zk-client-ca + namespace: default + autoGenerate: true +--- apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperZnode metadata: diff --git a/tests/templates/kuttl/smoke/02-assert.yaml b/tests/templates/kuttl/smoke/02-assert.yaml index c184732b..140232f1 100644 --- a/tests/templates/kuttl/smoke/02-assert.yaml +++ b/tests/templates/kuttl/smoke/02-assert.yaml @@ -5,3 +5,4 @@ metadata: name: test-regorule commands: - script: kubectl exec -n $NAMESPACE zk-test-helper-0 -- python /tmp/test_zookeeper.py -n $NAMESPACE + - script: kubectl exec -n $NAMESPACE test-zk-server-primary-0 -- /tmp/test_tls.sh $NAMESPACE diff --git a/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml b/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml index e0d3f04a..e41ef8fb 100644 --- a/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml +++ b/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml @@ -5,3 +5,4 @@ commands: - script: kubectl cp -n $NAMESPACE ./test_zookeeper.py zk-test-helper-0:/tmp - script: kubectl cp -n $NAMESPACE ./requirements.txt zk-test-helper-0:/tmp - script: kubectl exec -n $NAMESPACE zk-test-helper-0 -- pip install --user -r /tmp/requirements.txt + - script: kubectl cp -n $NAMESPACE ./test_tls.sh test-zk-server-primary-0:/tmp diff --git a/tests/templates/kuttl/smoke/test_tls.sh b/tests/templates/kuttl/smoke/test_tls.sh new file mode 100755 index 00000000..bf68c882 --- /dev/null +++ b/tests/templates/kuttl/smoke/test_tls.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Usage: test_tls.sh namespace + +NAMESPACE=$1 +SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2282" + +# just to be safe... +unset QUORUM_STORE_SECRET +unset CLIENT_STORE_SECRET +unset CLIENT_JVMFLAGS + +echo "Start TLS testing..." +############################################################################ +# Test the plaintext unsecured connection +############################################################################ +if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; +then + echo "[ERROR] Could not establish unsecure connection!" + exit 1 +fi +echo "[SUCCESS] Unsecure client connection established!" + +############################################################################ +# We set the correct client tls credentials and expect to be able to connect +############################################################################ +CLIENT_STORE_SECRET="$(< /stackable/rwconfig/zoo.cfg grep "ssl.keyStore.password" | cut -d "=" -f2)" +export CLIENT_STORE_SECRET +export CLIENT_JVMFLAGS=" +-Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider +-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty +-Dzookeeper.client.secure=true +-Dzookeeper.ssl.keyStore.location=/stackable/tls/client/keystore.p12 +-Dzookeeper.ssl.keyStore.password=${CLIENT_STORE_SECRET} +-Dzookeeper.ssl.trustStore.location=/stackable/tls/client/truststore.p12 +-Dzookeeper.ssl.trustStore.password=${CLIENT_STORE_SECRET}" + +if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; +then + echo "[ERROR] Could not establish secure connection using client certificates!" + exit 1 +fi +echo "[SUCCESS] Secure and authenticated client connection established!" + +############################################################################ +# We set the (wrong) quorum tls credentials and expect to fail (wrong certificate) +############################################################################ +QUORUM_STORE_SECRET="$(< /stackable/rwconfig/zoo.cfg grep "ssl.quorum.keyStore.password" | cut -d "=" -f2)" +export QUORUM_STORE_SECRET +export CLIENT_JVMFLAGS=" +-Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider +-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty +-Dzookeeper.client.secure=true +-Dzookeeper.ssl.keyStore.location=/stackable/tls/quorum/keystore.p12 +-Dzookeeper.ssl.keyStore.password=${QUORUM_STORE_SECRET} +-Dzookeeper.ssl.trustStore.location=/stackable/tls/quorum/truststore.p12 +-Dzookeeper.ssl.trustStore.password=${QUORUM_STORE_SECRET}" + +if /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; +then + echo "[ERROR] Could establish secure connection with quorum certificates (should not be happening)!" + exit 1 +fi +echo "[SUCCESS] Could not establish secure connection with (wrong) quorum certificates!" + +echo "All TLS tests successful!" +exit 0