From 57826c1969df4067a7991e18483c39b82073b019 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 3 May 2022 19:19:03 +0200 Subject: [PATCH 01/27] WIP --- Cargo.lock | 344 +++++++++++++++--- deploy/crd/zookeepercluster.crd.yaml | 22 ++ deploy/helm/zookeeper-operator/crds/crds.yaml | 22 ++ deploy/manifests/crds.yaml | 22 ++ examples/simple-zookeeper-tls-cluster.yaml | 52 +++ rust/crd/Cargo.toml | 3 +- rust/crd/src/lib.rs | 75 +++- rust/operator-binary/Cargo.toml | 8 +- rust/operator-binary/src/auth_config.rs | 114 ++++++ rust/operator-binary/src/config.rs | 15 + rust/operator-binary/src/discovery.rs | 4 +- rust/operator-binary/src/main.rs | 10 +- rust/operator-binary/src/zk_controller.rs | 61 +++- rust/operator-binary/src/znode_controller.rs | 7 +- 14 files changed, 683 insertions(+), 76 deletions(-) create mode 100644 examples/simple-zookeeper-tls-cluster.yaml create mode 100644 rust/operator-binary/src/auth_config.rs create mode 100644 rust/operator-binary/src/config.rs diff --git a/Cargo.lock b/Cargo.lock index 9249a32e..391df40b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,17 @@ version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +[[package]] +name = "async-trait" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -133,6 +144,12 @@ dependencies = [ "git2", ] +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "byteorder" version = "1.4.3" @@ -204,16 +221,16 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.8" +version = "3.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c" +checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db" dependencies = [ "atty", "bitflags", "clap_derive", + "clap_lex", "indexmap", "lazy_static", - "os_str_bytes", "strsim", "termcolor", "textwrap", @@ -232,6 +249,15 @@ dependencies = [ "syn", ] +[[package]] +name = "clap_lex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -277,6 +303,16 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.8", +] + [[package]] name = "crossbeam-deque" version = "0.7.4" @@ -284,7 +320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" dependencies = [ "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "maybe-uninit", ] @@ -296,7 +332,7 @@ checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg", "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "lazy_static", "maybe-uninit", "memoffset", @@ -310,7 +346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" dependencies = [ "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "maybe-uninit", ] @@ -325,6 +361,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if 1.0.0", + "lazy_static", +] + [[package]] name = "darling" version = "0.13.1" @@ -775,12 +821,30 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio 1.17.0", + "tokio 1.18.0", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper-openssl" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ee5d7a8f718585d1c3c61dfde28ef5b0bb14734b4db13f5ada856cdc6c612b" +dependencies = [ + "http", + "hyper", + "linked_hash_set", + "once_cell", + "openssl", + "openssl-sys", + "parking_lot 0.12.0", + "tokio 1.18.0", + "tokio-openssl", + "tower-layer", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -789,7 +853,7 @@ checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ "hyper", "pin-project-lite", - "tokio 1.17.0", + "tokio 1.18.0", "tokio-io-timeout", ] @@ -802,7 +866,7 @@ dependencies = [ "bytes 1.1.0", "hyper", "native-tls", - "tokio 1.17.0", + "tokio 1.18.0", "tokio-native-tls", ] @@ -842,6 +906,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "integer-encoding" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e85a1509a128c855368e135cffcde7eac17d8e1083f41e2b98c58bc1a5074be" + [[package]] name = "iovec" version = "0.1.4" @@ -877,6 +947,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "json-patch" version = "0.2.6" @@ -926,9 +1005,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcc72fdf0c491160a34d4a1bfb03f96da8a5054288d61c816d514b5c2fa49ea" +checksum = "342744dfeb81fe186b84f485b33f12c6a15d3396987d933b06a566a3db52ca38" dependencies = [ "k8s-openapi", "kube-client", @@ -939,9 +1018,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b01c722d55ffedec74cbc259b4508d8a59bf19540006ec87618f76ab156579" +checksum = "3f69a504997799340408635d6e351afb8aab2c34ca3165e162f41b3b34a69a79" dependencies = [ "base64", "bytes 1.1.0", @@ -952,6 +1031,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-openssl", "hyper-timeout", "hyper-tls", "jsonpath_lib", @@ -965,7 +1045,7 @@ dependencies = [ "serde_json", "serde_yaml", "thiserror", - "tokio 1.17.0", + "tokio 1.18.0", "tokio-native-tls", "tokio-util", "tower", @@ -975,9 +1055,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd9e3535777edd122cc26fe3fe6357066b33eff63d8b919862edbe7a956a679" +checksum = "a4a247487699941baaf93438d65b12d4e32450bea849d619d19ed394e8a4a645" dependencies = [ "chrono", "form_urlencoded", @@ -993,9 +1073,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1322e25c20dd6f18ca6baecc88bb130331e99d988df9d7a9a207f15819e05bff" +checksum = "203f7c5acf9d0dfb0b08d44ec1d66ace3d1dfe0cdd82e65e274f3f96615d666c" dependencies = [ "darling", "proc-macro2", @@ -1006,9 +1086,9 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816c8c086f8bbcf9a4db0b7a68db90b784ef6292a57de35c64cccb90d5edfbe5" +checksum = "02ea50e6ed56578e1d1d02548901b12fe6d3edbf110269a396955e285d487973" dependencies = [ "ahash", "backoff", @@ -1023,7 +1103,7 @@ dependencies = [ "serde_json", "smallvec 1.8.0", "thiserror", - "tokio 1.17.0", + "tokio 1.18.0", "tokio-util", "tracing", ] @@ -1070,6 +1150,15 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "linked_hash_set" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lock_api" version = "0.3.4" @@ -1323,6 +1412,60 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand", + "thiserror", + "tokio 1.18.0", + "tokio-stream", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c0b12cd9e3f9b35b52f6e0dac66866c519b26f424f4bbf96e3fe8bfbdc5229" +dependencies = [ + "async-trait", + "lazy_static", + "opentelemetry", + "opentelemetry-semantic-conventions", + "thiserror", + "thrift", + "tokio 1.18.0", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "ordered-float" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "2.10.0" @@ -1337,9 +1480,6 @@ name = "os_str_bytes" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" -dependencies = [ - "memchr", -] [[package]] name = "parking_lot" @@ -1484,8 +1624,8 @@ dependencies = [ [[package]] name = "product-config" -version = "0.3.1" -source = "git+https://github.com/stackabletech/product-config.git?tag=0.3.1#40c93e5283beef100c9fecdb6368f1e1480db3e8" +version = "0.4.0" +source = "git+https://github.com/stackabletech/product-config.git?tag=0.4.0#e1e5938b4f6120f85a088194e86d22433fdba731" dependencies = [ "fancy-regex", "java-properties", @@ -1737,7 +1877,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" dependencies = [ - "ordered-float", + "ordered-float 2.10.0", "serde", ] @@ -1866,8 +2006,7 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.15.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.15.0#c7c408d476c0b7ba06833c19e5c9d2aefa5875ae" +version = "0.17.0" dependencies = [ "backoff", "chrono", @@ -1880,6 +2019,8 @@ dependencies = [ "k8s-openapi", "kube", "lazy_static", + "opentelemetry", + "opentelemetry-jaeger", "product-config", "rand", "regex", @@ -1889,8 +2030,9 @@ dependencies = [ "serde_yaml", "strum", "thiserror", - "tokio 1.17.0", + "tokio 1.18.0", "tracing", + "tracing-opentelemetry", "tracing-subscriber", ] @@ -1925,7 +2067,7 @@ dependencies = [ "stackable-zookeeper-crd", "strum", "tokio 0.1.22", - "tokio 1.17.0", + "tokio 1.18.0", "tokio-executor", "tokio-zookeeper", "tracing", @@ -2040,6 +2182,28 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b82ca8f46f95b3ce96081fe3dd89160fdea970c254bb72925255d1b62aae692e" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float 1.1.1", + "threadpool", +] + [[package]] name = "time" version = "0.1.43" @@ -2091,9 +2255,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "0f48b6d60512a392e34dbf7fd456249fd2de3c83669ab642e021903f4015185b" dependencies = [ "bytes 1.1.0", "libc", @@ -2136,7 +2300,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.31", ] @@ -2169,7 +2333,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ "pin-project-lite", - "tokio 1.17.0", + "tokio 1.18.0", ] [[package]] @@ -2190,7 +2354,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.17.0", + "tokio 1.18.0", +] + +[[package]] +name = "tokio-openssl" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +dependencies = [ + "futures-util", + "openssl", + "openssl-sys", + "tokio 1.18.0", ] [[package]] @@ -2199,7 +2375,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.31", "lazy_static", "log", @@ -2212,6 +2388,17 @@ dependencies = [ "tokio-sync", ] +[[package]] +name = "tokio-stream" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio 1.18.0", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -2244,7 +2431,7 @@ checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" dependencies = [ "crossbeam-deque", "crossbeam-queue", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.31", "lazy_static", "log", @@ -2259,7 +2446,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.31", "slab", "tokio-executor", @@ -2310,7 +2497,7 @@ dependencies = [ "log", "pin-project-lite", "slab", - "tokio 1.17.0", + "tokio 1.18.0", ] [[package]] @@ -2346,7 +2533,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio 1.17.0", + "tokio 1.18.0", "tokio-util", "tower-layer", "tower-service", @@ -2387,9 +2574,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80b9fa4360528139bc96100c160b7ae879f5567f49f1782b0b02035b0358ebf3" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if 1.0.0", "log", @@ -2430,11 +2617,24 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f9378e96a9361190ae297e7f3a8ff644aacd2897f244b1ff81f381669196fa6" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-subscriber" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0ab7bdc962035a87fba73f3acca9b8a8d0034c2e6f60b84aeaaddddc155dce" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" dependencies = [ "ansi_term", "lazy_static", @@ -2542,6 +2742,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + [[package]] name = "winapi" version = "0.2.8" diff --git a/deploy/crd/zookeepercluster.crd.yaml b/deploy/crd/zookeepercluster.crd.yaml index f90256cd..070b2cf4 100644 --- a/deploy/crd/zookeepercluster.crd.yaml +++ b/deploy/crd/zookeepercluster.crd.yaml @@ -23,6 +23,28 @@ spec: spec: description: A cluster of ZooKeeper nodes properties: + config: + nullable: true + properties: + clientAuthentication: + nullable: true + properties: + authenticationClass: + type: string + required: + - authenticationClass + type: object + quorumTlsSecretClass: + type: string + tls: + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + type: object servers: nullable: true properties: diff --git a/deploy/helm/zookeeper-operator/crds/crds.yaml b/deploy/helm/zookeeper-operator/crds/crds.yaml index 5a59580a..ec9c30ac 100644 --- a/deploy/helm/zookeeper-operator/crds/crds.yaml +++ b/deploy/helm/zookeeper-operator/crds/crds.yaml @@ -25,6 +25,28 @@ spec: spec: description: A cluster of ZooKeeper nodes properties: + config: + nullable: true + properties: + clientAuthentication: + nullable: true + properties: + authenticationClass: + type: string + required: + - authenticationClass + type: object + quorumTlsSecretClass: + type: string + tls: + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + type: object servers: nullable: true properties: diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index f9dbdd71..483f10d2 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -26,6 +26,28 @@ spec: spec: description: A cluster of ZooKeeper nodes properties: + config: + nullable: true + properties: + clientAuthentication: + nullable: true + properties: + authenticationClass: + type: string + required: + - authenticationClass + type: object + quorumTlsSecretClass: + type: string + tls: + nullable: true + properties: + secretClass: + type: string + required: + - secretClass + type: object + type: object servers: nullable: true properties: diff --git a/examples/simple-zookeeper-tls-cluster.yaml b/examples/simple-zookeeper-tls-cluster.yaml new file mode 100644 index 00000000..6fc1a3dc --- /dev/null +++ b/examples/simple-zookeeper-tls-cluster.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: simple-zk +spec: + version: 3.8.0 + config: + tls: + secretClass: tls + clientAuthentication: + authenticationClass: zookeeper-client-tls + quorumTlsSecretClass: tls + servers: + roleGroups: + default: + replicas: 3 +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: zookeeper-client-tls +spec: + provider: + tls: + verification: + server: + caCert: + secretClass: tls +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: tls +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-ca + namespace: default + autoGenerate: true +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: simple-znode +spec: + clusterRef: + name: simple-zk + # Optional when ZookeeperZnode is in the same Namespace as the ZookeeperCluster + # namespace: default \ No newline at end of file diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index f3298c47..14f94c72 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -11,5 +11,6 @@ repository = "https://github.com/stackabletech/zookeeper-operator" serde = "1.0.136" serde_json = "1.0.79" snafu = "0.7.0" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +#stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.18.0" } +stackable-operator = { path = "../../../operator-rs"} strum = { version = "0.24.0", features = ["derive"] } diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 40e557dc..145cb275 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use snafu::{OptionExt, Snafu}; use stackable_operator::{ @@ -9,6 +7,10 @@ use stackable_operator::{ role_utils::{Role, RoleGroupRef}, schemars::{self, JsonSchema}, }; +use std::collections::BTreeMap; + +pub const APP_PORT: u16 = 2181; +pub const SECURE_APP_PORT: u16 = 2281; /// A cluster of ZooKeeper nodes #[derive(Clone, CustomResource, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -36,6 +38,43 @@ pub struct ZookeeperClusterSpec { pub version: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub servers: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config: Option, +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GlobalZookeeperConfig { + #[serde(flatten)] + pub tls_config: Option, +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TlsConfig { + pub tls: Option, + pub client_authentication: Option, + pub quorum_tls_secret_class: String, +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TlsSecretClass { + pub secret_class: String, +} + +impl Default for TlsSecretClass { + fn default() -> Self { + Self { + secret_class: "tls".to_string(), + } + } +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ClientAuthenticationClass { + pub authentication_class: String, } #[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -179,6 +218,38 @@ impl ZookeeperCluster { }) })) } + + pub fn client_port(&self) -> u16 { + if self.client_auth().is_some() { + SECURE_APP_PORT + } else { + APP_PORT + } + } + + pub fn tls_secret_class(&self) -> Option<&TlsSecretClass> { + let spec: &ZookeeperClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.tls_config.as_ref()) + .and_then(|tls| tls.tls.as_ref()) + } + + pub fn client_auth(&self) -> Option<&ClientAuthenticationClass> { + let spec: &ZookeeperClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.tls_config.as_ref()) + .and_then(|tls| tls.client_authentication.as_ref()) + } + + pub fn quorum_tls_secret_class(&self) -> Option<&String> { + let spec: &ZookeeperClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.tls_config.as_ref()) + .map(|tls| &tls.quorum_tls_secret_class) + } } /// Reference to a single `Pod` that is a component of a [`ZookeeperCluster`] diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 3ceec06c..16fbff5b 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -19,16 +19,18 @@ serde_json = "1.0.79" serde_yaml = "0.8.23" snafu = "0.7.0" strum = { version = "0.24.0", features = ["derive"] } -tokio = { version = "1.17.0", features = ["full"] } +tokio = { version = "1.18.0", features = ["full"] } tokio01 = { version = "0.1.22", package = "tokio" } tokio-executor = "0.1.10" tokio-zookeeper = "0.1.3" tracing = "0.1.33" pin-project = "1.0.10" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +#stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.18.0" } +stackable-operator = { path = "../../../operator-rs"} stackable-zookeeper-crd = { path = "../crd" } [build-dependencies] built = { version = "0.5.1", features = ["chrono", "git2"] } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +#stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.18.0" } +stackable-operator = { path = "../../../operator-rs"} stackable-zookeeper-crd = { path = "../crd" } diff --git a/rust/operator-binary/src/auth_config.rs b/rust/operator-binary/src/auth_config.rs new file mode 100644 index 00000000..e495ba69 --- /dev/null +++ b/rust/operator-binary/src/auth_config.rs @@ -0,0 +1,114 @@ +use stackable_operator::builder::{ContainerBuilder, PodBuilder}; +use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperClusterSpec}; +use std::collections::BTreeMap; + +pub fn xyz(zk: &ZookeeperCluster, cb: &mut ContainerBuilder, pb: &mut PodBuilder) { + let spec: &ZookeeperClusterSpec = &zk.spec; + + //spec.config.map(|c| c.tls_config.map(|t| t.tls)) +} + +pub fn quorum_auth_config_for_cm( + product_config: &mut BTreeMap, + directory: &str, + password: &str, +) { + product_config.extend(quorum_auth_properties(directory, password)); +} + +pub fn client_auth_config_for_cm( + product_config: &mut BTreeMap, + zk: &ZookeeperCluster, + directory: &str, + password: &str, +) { + // Remove clientPort in favor for secureClientPort + product_config.remove("clientPort"); + product_config.extend(client_auth_properties(zk, directory, password)); +} + +pub fn create_key_and_trust_store_cmd(directory: &str, password: &str) -> Vec { + vec![ + "echo Storing password".to_string(), + format!("echo {pw} > {dir}/password", pw = password, dir = directory), + "echo Cleaning up truststore - just in case".to_string(), + format!("rm -f {dir}/truststore.p12", dir = directory), + "echo Creating truststore".to_string(), + format!("keytool -importcert -file {dir}/ca.crt -keystore {dir}/truststore.p12 -storetype pkcs12 -noprompt -alias ca_cert -storepass {pw}", pw = password, dir = directory), + "echo Creating keystore".to_string(), + format!("openssl pkcs12 -export -in {dir}/chain.crt -inkey {dir}/tls.key -out {dir}/keystore.p12 --passout file:{dir}/password", + dir = directory), + "echo Cleaning up password".to_string(), + format!("rm -f {dir}/password", dir = directory), + "echo chowning store directory".to_string(), + format!("chown -R stackable:stackable {dir}", dir = directory), + "echo chmodding store directory".to_string(), + format!("chmod -R a=,u=rwX {dir}", dir = directory), + ] +} + +fn quorum_auth_properties(directory: &str, password: &str) -> BTreeMap { + vec![ + ("sslQuorum".to_string(), "true".to_string()), + ( + "serverCnxnFactory".to_string(), + "org.apache.zookeeper.server.NettyServerCnxnFactory".to_string(), + ), + ( + "ssl.quorum.keyStore.location".to_string(), + format!("{dir}/keystore.p12", dir = directory), + ), + ( + "ssl.quorum.keyStore.password".to_string(), + password.to_string(), + ), + ( + "ssl.quorum.trustStore.location".to_string(), + format!("{dir}/truststore.p12", dir = directory), + ), + ( + "ssl.quorum.trustStore.password".to_string(), + password.to_string(), + ), + ( + "ssl.quorum.hostnameVerification".to_string(), + "true".to_string(), + ), + ( + "authProvider.x509".to_string(), + "org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string(), + ), + ] + .into_iter() + .collect::>() +} + +fn client_auth_properties( + zk: &ZookeeperCluster, + directory: &str, + password: &str, +) -> BTreeMap { + vec![ + ("secureClientPort".to_string(), zk.client_port().to_string()), + ( + "serverCnxnFactory".to_string(), + "org.apache.zookeeper.server.NettyServerCnxnFactory".to_string(), + ), + ( + "ssl.keyStore.location".to_string(), + format!("{dir}/keystore.p12", dir = directory), + ), + ("ssl.keyStore.password".to_string(), password.to_string()), + ( + "ssl.trustStore.location".to_string(), + format!("{dir}/truststore.p12", dir = directory), + ), + ("ssl.trustStore.password".to_string(), password.to_string()), + ( + "authProvider.x509".to_string(), + "org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string(), + ), + ] + .into_iter() + .collect::>() +} diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs new file mode 100644 index 00000000..db5385a8 --- /dev/null +++ b/rust/operator-binary/src/config.rs @@ -0,0 +1,15 @@ +use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperClusterSpec}; + +pub fn build_init_container_args(zk: &ZookeeperCluster) -> Vec { + //let mut args = vec![]; + let spec: &ZookeeperClusterSpec = &zk.spec; + + vec![ + "chown stackable:stackable /stackable/data", + "chmod a=,u=rwX /stackable/data", + "expr $MYID_OFFSET + $(echo $POD_NAME | sed 's/.*-//') > /stackable/data/myid", + ] + .join(" && "); + + vec!["test".to_string()] +} diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index f0a0ab6b..4af2d659 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -8,7 +8,7 @@ use stackable_operator::{ }; use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperRole}; -use crate::{zk_controller::zk_version, APP_NAME, APP_PORT}; +use crate::{zk_controller::zk_version, APP_NAME}; #[derive(Snafu, Debug)] pub enum Error { @@ -122,7 +122,7 @@ fn pod_hosts(zk: &ZookeeperCluster) -> Result anyhow::Result<()> { - stackable_operator::logging::initialize_logging("ZOOKEEPER_OPERATOR_LOG"); // tokio-zookeeper depends on Tokio 0.1 let tokio01_runtime = tokio01::runtime::Runtime::new()?; @@ -51,7 +51,13 @@ async fn main() -> anyhow::Result<()> { Command::Run(ProductOperatorRun { product_config, watch_namespace, + tracing_target, }) => { + stackable_operator::logging::initialize_logging( + "ZOOKEEPER_OPERATOR_LOG", + APP_NAME, + tracing_target, + ); stackable_operator::utils::print_startup_string( built_info::PKG_DESCRIPTION, built_info::PKG_VERSION, diff --git a/rust/operator-binary/src/zk_controller.rs b/rust/operator-binary/src/zk_controller.rs index 4dec5e81..5fbcf754 100644 --- a/rust/operator-binary/src/zk_controller.rs +++ b/rust/operator-binary/src/zk_controller.rs @@ -8,12 +8,15 @@ use std::{ time::Duration, }; +use crate::config::build_init_container_args; use crate::{ + auth_config::{self, create_key_and_trust_store_cmd}, discovery::{self, build_discovery_configmaps}, - APP_NAME, APP_PORT, + APP_NAME, }; use fnv::FnvHasher; use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::k8s_openapi::api::core::v1::CSIVolumeSource; use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, k8s_openapi::{ @@ -44,6 +47,11 @@ use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperClusterStatus, Zookeepe use strum::{EnumDiscriminants, IntoStaticStr}; const FIELD_MANAGER_SCOPE: &str = "zookeepercluster"; +const QUORUM_TLS_DIR: &str = "/stackable/tls/quorum"; +const CLIENT_TLS_DIR: &str = "/stackable/tls/client"; +// TODO: Generate? Currently this is written in the ConfigMap. +// It probably makes sense to add that after mounting the CM? +const TLS_STORE_SECRET: &str = "MyS3cr4T"; pub struct Ctx { pub client: stackable_operator::client::Client, @@ -157,6 +165,7 @@ pub async fn reconcile_zk( false, ) .context(InvalidProductConfigSnafu)?; + let role_server_config = validated_config .get(&ZookeeperRole::Server.to_string()) .map(Cow::Borrowed) @@ -247,7 +256,7 @@ pub fn build_server_role_service(zk: &ZookeeperCluster) -> Result { spec: Some(ServiceSpec { ports: Some(vec![ServicePort { name: Some("zk".to_string()), - port: APP_PORT.into(), + port: zk.client_port().into(), protocol: Some("TCP".to_string()), ..ServicePort::default() }]), @@ -272,9 +281,12 @@ fn build_server_rolegroup_config_map( zoo_cfg.extend(zk.pods().into_iter().flatten().map(|pod| { ( format!("server.{}", pod.zookeeper_myid), - format!("{}:2888:3888;{}", pod.fqdn(), APP_PORT), + format!("{}:2888:3888;{}", pod.fqdn(), zk.client_port()), ) })); + + // TLS + let zoo_cfg = zoo_cfg .into_iter() .map(|(k, v)| (k, Some(v))) @@ -336,7 +348,7 @@ fn build_server_rolegroup_service( ports: Some(vec![ ServicePort { name: Some("zk".to_string()), - port: APP_PORT.into(), + port: zk.client_port().into(), protocol: Some("TCP".to_string()), ..ServicePort::default() }, @@ -390,18 +402,11 @@ fn build_server_rolegroup_statefulset( ..EnvVar::default() }) .collect::>(); + let mut container_prepare = ContainerBuilder::new("prepare") .image(&image) - .args(vec![ - "sh".to_string(), - "-c".to_string(), - [ - "chown stackable:stackable /stackable/data", - "chmod a=,u=rwX /stackable/data", - "expr $MYID_OFFSET + $(echo $POD_NAME | sed 's/.*-//') > /stackable/data/myid", - ] - .join(" && "), - ]) + .command(vec!["sh".to_string(), "-c".to_string()]) + .args(build_init_container_args(zk)) .add_env_vars(env.clone()) .add_env_vars(vec![EnvVar { name: "POD_NAME".to_string(), @@ -415,6 +420,7 @@ fn build_server_rolegroup_statefulset( ..EnvVar::default() }]) .add_volume_mount("data", "/stackable/data") + .add_volume_mount("quorum-tls", QUORUM_TLS_DIR) .build(); container_prepare .security_context @@ -439,19 +445,20 @@ fn build_server_rolegroup_statefulset( // we can use Bash's virtual /dev/tcp filesystem to accomplish the same thing format!( "exec 3<>/dev/tcp/localhost/{} && echo srvr >&3 && grep '^Mode: ' <&3", - APP_PORT + zk.client_port() ), ]), }), period_seconds: Some(1), ..Probe::default() }) - .add_container_port("zk", APP_PORT.into()) + .add_container_port("zk", zk.client_port().into()) .add_container_port("zk-leader", 2888) .add_container_port("zk-election", 3888) .add_container_port("metrics", 9505) .add_volume_mount("data", "/stackable/data") .add_volume_mount("config", "/stackable/config") + .add_volume_mount("quorum-tls", QUORUM_TLS_DIR) .build(); Ok(StatefulSet { metadata: ObjectMetaBuilder::new() @@ -504,6 +511,28 @@ fn build_server_rolegroup_statefulset( }), ..Volume::default() }) + .add_volume(Volume { + name: "quorum-tls".to_string(), + csi: Some(CSIVolumeSource { + driver: "secrets.stackable.tech".to_string(), + volume_attributes: Some( + vec![ + ( + "secrets.stackable.tech/class".to_string(), + "tls".to_string(), + ), + ( + "secrets.stackable.tech/scope".to_string(), + "node,pod".to_string(), + ), + ] + .into_iter() + .collect::>(), + ), + ..CSIVolumeSource::default() + }), + ..Volume::default() + }) .security_context(PodSecurityContext { fs_group: Some(1000), ..PodSecurityContext::default() diff --git a/rust/operator-binary/src/znode_controller.rs b/rust/operator-binary/src/znode_controller.rs index 753fd5d6..811053f3 100644 --- a/rust/operator-binary/src/znode_controller.rs +++ b/rust/operator-binary/src/znode_controller.rs @@ -4,10 +4,7 @@ use std::{convert::Infallible, sync::Arc, time::Duration}; -use crate::{ - discovery::{self, build_discovery_configmaps}, - APP_PORT, -}; +use crate::discovery::{self, build_discovery_configmaps}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ k8s_openapi::api::core::v1::{ConfigMap, Service}, @@ -244,7 +241,7 @@ fn zk_mgmt_addr(zk: &ZookeeperCluster) -> Result { .with_context(|| NoZkFqdnSnafu { zk: ObjectRef::from_obj(zk), })?, - APP_PORT, + zk.client_port(), )) } From 90807b6bf52c70cde86db6722c03642b5c9c4792 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 4 May 2022 16:26:16 +0200 Subject: [PATCH 02/27] adapted product config properties --- deploy/config-spec/properties.yaml | 235 +++++++++++++++++++++++++---- 1 file changed, 207 insertions(+), 28 deletions(-) diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index aebefcde..8fd08a27 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -9,6 +9,7 @@ spec: - unit: &unitPort name: "port" regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" + properties: - property: &tickTime propertyNames: @@ -29,28 +30,6 @@ properties: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "The basic time unit in milliseconds used by ZooKeeper. It is used to do heartbeats and the minimum session timeout will be twice the tickTime." - - property: &clientPort - propertyNames: - - name: "clientPort" - kind: - type: "file" - file: "zoo.cfg" - datatype: - type: "integer" - unit: *unitPort - min: "1024" - max: "65535" - defaultValues: - - value: "2181" - recommendedValues: - - value: "2181" - roles: - - name: "server" - required: true - asOfVersion: "0.0.0" - comment: "See QuorumPeerConfig.java, I'm unsure what happens when this is set to 0, it might work, it might not" - description: "The port to listen for client connections; that is, the port that clients attempt to connect to." - - property: &dataDir propertyNames: - name: "dataDir" @@ -109,11 +88,34 @@ properties: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "Amount of time, in ticks (see `tickTime`), to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they will be dropped. In other words: The number of ticks that can pass between sending a request and getting an acknowledgment before a follower is dropped." - - property: &metricsPort + ########################## + # Ports + ########################## + - property: &clientPort + propertyNames: + - name: "clientPort" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "integer" + unit: *unitPort + min: "1024" + max: "65535" + defaultValues: + - value: "2181" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "The plaintext port to listen on for client connections." + + - property: &secureClientPort propertyNames: - - name: "metricsPort" + - name: "secureClientPort" kind: - type: "env" + type: "file" + file: "zoo.cfg" datatype: type: "integer" unit: *unitPort @@ -123,9 +125,9 @@ properties: - name: "server" required: false asOfVersion: "0.0.0" - description: "The port where ZooKeeper metrics are exposed as a Prometheus endpoint." + description: "The secure port to listen on for secure client connections using TLS/SSL." - - property: &admin_serverPort + - property: &adminServerPort propertyNames: - name: "admin.serverPort" kind: @@ -142,4 +144,181 @@ properties: - name: "server" required: true asOfVersion: "0.0.0" - description: "The zookeeper admin server port." \ No newline at end of file + description: "The port the embedded Jetty server listens on." + + ########################## + # Quorum TLS + ########################## + - property: &sslQuorum + propertyNames: + - name: "sslQuorum" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "boolean" + defaultValues: + - value: "false" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Enables encrypted quorum communication." + + - property: &sslQuorumKeyStoreLocation + propertyNames: + - name: "ssl.quorum.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumKeyStorePassword + propertyNames: + - name: "ssl.quorum.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum keystore." + + - property: &sslQuorumTrustStoreLocation + propertyNames: + - name: "ssl.quorum.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumTrustStorePassword + propertyNames: + - name: "ssl.quorum.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum truststore." + + ########################## + # Client TLS + ########################## + - property: &sslKeyStoreLocation + propertyNames: + - name: "ssl.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for client TLS connections." + + - property: &sslKeyStorePassword + propertyNames: + - name: "ssl.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client keystore." + + - property: &sslTrustStoreLocation + propertyNames: + - name: "ssl.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the remote credentials to be used for client TLS connections." + + - property: &sslTrustStorePassword + propertyNames: + - name: "ssl.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client truststore." + + - property: &authProviderX509 + propertyNames: + - name: "authProvider.x509" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "Used for secure client authentication.." + + ########################## + # Common TLS + ########################## + - property: &serverCnxnFactory + propertyNames: + - name: "serverCnxnFactory" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "This should be set to NettyServerCnxnFactory in order to use TLS based server communication." From 7b96ae8715550059f0a02a3bb52aaf6564f04155 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 4 May 2022 17:38:51 +0200 Subject: [PATCH 03/27] fixed boolean -> bool --- deploy/config-spec/properties.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index 8fd08a27..a18e87b1 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -156,7 +156,7 @@ properties: type: "file" file: "zoo.cfg" datatype: - type: "boolean" + type: "bool" defaultValues: - value: "false" roles: From 32aada006713fd6d31e8c8986201bdd2e445f182 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 4 May 2022 19:07:23 +0200 Subject: [PATCH 04/27] quorum and client secured. Works but needs cleanup and configuration settings (secret and authentication class hard coded atm) --- examples/simple-zookeeper-tls-cluster.yaml | 13 -- rust/crd/src/lib.rs | 150 ++++++++++++++++- rust/operator-binary/src/auth_config.rs | 134 +++++---------- rust/operator-binary/src/config.rs | 15 -- rust/operator-binary/src/main.rs | 1 - rust/operator-binary/src/zk_controller.rs | 179 +++++++++++++-------- 6 files changed, 295 insertions(+), 197 deletions(-) delete mode 100644 rust/operator-binary/src/config.rs diff --git a/examples/simple-zookeeper-tls-cluster.yaml b/examples/simple-zookeeper-tls-cluster.yaml index 6fc1a3dc..884fc9c8 100644 --- a/examples/simple-zookeeper-tls-cluster.yaml +++ b/examples/simple-zookeeper-tls-cluster.yaml @@ -28,19 +28,6 @@ spec: caCert: secretClass: tls --- -apiVersion: secrets.stackable.tech/v1alpha1 -kind: SecretClass -metadata: - name: tls -spec: - backend: - autoTls: - ca: - secret: - name: secret-provisioner-tls-ca - namespace: default - autoGenerate: true ---- apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperZnode metadata: diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 145cb275..f6c2b108 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -9,8 +9,17 @@ use stackable_operator::{ }; use std::collections::BTreeMap; -pub const APP_PORT: u16 = 2181; -pub const SECURE_APP_PORT: u16 = 2281; +pub const CLIENT_PORT: u16 = 2181; +pub const SECURE_CLIENT_PORT: u16 = 2282; + +pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; +pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; + +pub const QUORUM_TLS_DIR: &str = "/stackable/tls/quorum"; +pub const CLIENT_TLS_DIR: &str = "/stackable/tls/client"; +// TODO: Generate? Currently this is written in the ConfigMap. +// It probably makes sense to add that after mounting the CM? +pub const TLS_STORE_SECRET: &str = "My_5t4ck4ble_53cr4T"; /// A cluster of ZooKeeper nodes #[derive(Clone, CustomResource, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -90,9 +99,27 @@ impl ZookeeperConfig { pub const INIT_LIMIT: &'static str = "initLimit"; pub const SYNC_LIMIT: &'static str = "syncLimit"; pub const TICK_TIME: &'static str = "tickTime"; + pub const DATA_DIR: &'static str = "dataDir"; pub const MYID_OFFSET: &'static str = "MYID_OFFSET"; pub const SERVER_JVMFLAGS: &'static str = "SERVER_JVMFLAGS"; + pub const CLIENT_PORT: &'static str = "clientPort"; + + // Quorum TLS + pub const SSL_QUORUM: &'static str = "sslQuorum"; + pub const SSL_QUORUM_KEY_STORE_LOCATION: &'static str = "ssl.quorum.keyStore.location"; + pub const SSL_QUORUM_KEY_STORE_PASSWORD: &'static str = "ssl.quorum.keyStore.password"; + pub const SSL_QUORUM_TRUST_STORE_LOCATION: &'static str = "ssl.quorum.trustStore.location"; + pub const SSL_QUORUM_TRUST_STORE_PASSWORD: &'static str = "ssl.quorum.trustStore.password"; + // Client TLS + pub const SECURE_CLIENT_PORT: &'static str = "secureClientPort"; + pub const SSL_KEY_STORE_LOCATION: &'static str = "ssl.keyStore.location"; + pub const SSL_KEY_STORE_PASSWORD: &'static str = "ssl.keyStore.password"; + pub const SSL_TRUST_STORE_LOCATION: &'static str = "ssl.trustStore.location"; + pub const SSL_TRUST_STORE_PASSWORD: &'static str = "ssl.trustStore.password"; + pub const SSL_AUTH_PROVIDER_X509: &'static str = "authProvider.x509"; + // Common TLS + pub const SERVER_CNXN_FACTORY: &'static str = "serverCnxnFactory"; fn myid_offset(&self) -> u16 { self.myid_offset.unwrap_or(1) @@ -128,7 +155,7 @@ impl Configuration for ZookeeperConfig { fn compute_files( &self, - _resource: &Self::Configurable, + resource: &Self::Configurable, _role_name: &str, _file: &str, ) -> Result>, ConfigError> { @@ -142,6 +169,102 @@ impl Configuration for ZookeeperConfig { if let Some(tick_time) = self.tick_time { result.insert(Self::TICK_TIME.to_string(), Some(tick_time.to_string())); } + result.insert( + Self::DATA_DIR.to_string(), + Some(STACKABLE_DATA_DIR.to_string()), + ); + + // Quorum TLS + if resource.is_quorum_secure() { + result.insert(Self::SSL_QUORUM.to_string(), Some("true".to_string())); + result.insert( + Self::SERVER_CNXN_FACTORY.to_string(), + Some("org.apache.zookeeper.server.NettyServerCnxnFactory".to_string()), + ); + result.insert( + Self::SSL_QUORUM_KEY_STORE_LOCATION.to_string(), + Some(format!("{dir}/keystore.p12", dir = QUORUM_TLS_DIR)), + ); + result.insert( + Self::SSL_QUORUM_KEY_STORE_PASSWORD.to_string(), + Some(TLS_STORE_SECRET.to_string()), + ); + result.insert( + Self::SSL_QUORUM_TRUST_STORE_LOCATION.to_string(), + Some(format!("{dir}/truststore.p12", dir = QUORUM_TLS_DIR)), + ); + result.insert( + Self::SSL_QUORUM_TRUST_STORE_PASSWORD.to_string(), + Some(TLS_STORE_SECRET.to_string()), + ); + } + // Client TLS + if resource.is_client_secure() { + // We set only the clientPort and portUnification here because otherwise there is a port bind exception + // See: https://issues.apache.org/jira/browse/ZOOKEEPER-4276 + // --> Normally we would like to only set the secureClientPort (check out commented code below) + // What we tried: + // 1) Set clientPort and secureClientPort will fail with + // "static.config different from dynamic config .. " + // result.insert( + // Self::CLIENT_PORT.to_string(), + // Some(CLIENT_PORT.to_string()), + // ); + // result.insert( + // Self::SECURE_CLIENT_PORT.to_string(), + // Some(SECURE_CLIENT_PORT.to_string()), + // ); + + // 2) Setting only secureClientPort will result in the above mentioned bind exception + // The NettyFactory tries to bind multiple times on the secureClientPort. + // result.insert( + // Self::SECURE_CLIENT_PORT.to_string(), + // Some(resource.client_port().to_string()), + // ); + + // 3) Using the clientPort and portUnification still allows plaintext connection without + // authentication, but at least TLS and authentication works when connecting securely. + result.insert( + Self::CLIENT_PORT.to_string(), + Some(resource.client_port().to_string()), + ); + result.insert( + "client.portUnification".to_string(), + Some("true".to_string()), + ); + // END TICKET + + result.insert( + Self::SERVER_CNXN_FACTORY.to_string(), + Some("org.apache.zookeeper.server.NettyServerCnxnFactory".to_string()), + ); + result.insert( + Self::SSL_KEY_STORE_LOCATION.to_string(), + Some(format!("{dir}/keystore.p12", dir = CLIENT_TLS_DIR)), + ); + result.insert( + Self::SSL_KEY_STORE_PASSWORD.to_string(), + Some(TLS_STORE_SECRET.to_string()), + ); + result.insert( + Self::SSL_TRUST_STORE_LOCATION.to_string(), + Some(format!("{dir}/truststore.p12", dir = CLIENT_TLS_DIR)), + ); + result.insert( + Self::SSL_TRUST_STORE_PASSWORD.to_string(), + Some(TLS_STORE_SECRET.to_string()), + ); + result.insert( + Self::SSL_AUTH_PROVIDER_X509.to_string(), + Some("org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string()), + ); + } else { + result.insert( + Self::CLIENT_PORT.to_string(), + Some(resource.client_port().to_string()), + ); + } + Ok(result) } } @@ -221,12 +344,29 @@ impl ZookeeperCluster { pub fn client_port(&self) -> u16 { if self.client_auth().is_some() { - SECURE_APP_PORT + SECURE_CLIENT_PORT } else { - APP_PORT + CLIENT_PORT } } + pub fn is_quorum_secure(&self) -> bool { + let spec: &ZookeeperClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.tls_config.as_ref()) + .is_some() + } + + pub fn is_client_secure(&self) -> bool { + let spec: &ZookeeperClusterSpec = &self.spec; + spec.config + .as_ref() + .and_then(|c| c.tls_config.as_ref()) + .and_then(|tls| tls.client_authentication.as_ref()) + .is_some() + } + pub fn tls_secret_class(&self) -> Option<&TlsSecretClass> { let spec: &ZookeeperClusterSpec = &self.spec; spec.config diff --git a/rust/operator-binary/src/auth_config.rs b/rust/operator-binary/src/auth_config.rs index e495ba69..63732c42 100644 --- a/rust/operator-binary/src/auth_config.rs +++ b/rust/operator-binary/src/auth_config.rs @@ -1,114 +1,54 @@ -use stackable_operator::builder::{ContainerBuilder, PodBuilder}; -use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperClusterSpec}; -use std::collections::BTreeMap; +use stackable_zookeeper_crd::{ + ZookeeperCluster, CLIENT_TLS_DIR, QUORUM_TLS_DIR, STACKABLE_DATA_DIR, TLS_STORE_SECRET, +}; -pub fn xyz(zk: &ZookeeperCluster, cb: &mut ContainerBuilder, pb: &mut PodBuilder) { - let spec: &ZookeeperClusterSpec = &zk.spec; +pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { + let mut args = vec![]; + // TODO: we need to check if another secret / authentication class is used + if zk.is_quorum_secure() { + args.extend(create_key_and_trust_store_cmd( + QUORUM_TLS_DIR, + TLS_STORE_SECRET, + )); + } - //spec.config.map(|c| c.tls_config.map(|t| t.tls)) -} + if zk.is_client_secure() { + args.extend(create_key_and_trust_store_cmd( + CLIENT_TLS_DIR, + TLS_STORE_SECRET, + )); + } -pub fn quorum_auth_config_for_cm( - product_config: &mut BTreeMap, - directory: &str, - password: &str, -) { - product_config.extend(quorum_auth_properties(directory, password)); -} + args.extend([ + format!("chown stackable:stackable {dir}", dir = STACKABLE_DATA_DIR), + format!("chmod a=,u=rwX {dir}", dir = STACKABLE_DATA_DIR), + format!( + "expr $MYID_OFFSET + $(echo $POD_NAME | sed 's/.*-//') > {dir}/myid", + dir = STACKABLE_DATA_DIR + ), + ]); -pub fn client_auth_config_for_cm( - product_config: &mut BTreeMap, - zk: &ZookeeperCluster, - directory: &str, - password: &str, -) { - // Remove clientPort in favor for secureClientPort - product_config.remove("clientPort"); - product_config.extend(client_auth_properties(zk, directory, password)); + args.join(" && ") } -pub fn create_key_and_trust_store_cmd(directory: &str, password: &str) -> Vec { +fn create_key_and_trust_store_cmd(directory: &str, password: &str) -> Vec { vec![ - "echo Storing password".to_string(), + format!("echo [{dir}] Storing password", dir = directory), format!("echo {pw} > {dir}/password", pw = password, dir = directory), - "echo Cleaning up truststore - just in case".to_string(), + format!("echo [{dir}] Cleaning up truststore - just in case", dir = directory), format!("rm -f {dir}/truststore.p12", dir = directory), - "echo Creating truststore".to_string(), + format!("echo [{dir}] Creating truststore", dir = directory), format!("keytool -importcert -file {dir}/ca.crt -keystore {dir}/truststore.p12 -storetype pkcs12 -noprompt -alias ca_cert -storepass {pw}", pw = password, dir = directory), - "echo Creating keystore".to_string(), + format!("echo [{dir}] Creating certificate chain", dir = directory), + format!("cat {dir}/ca.crt {dir}/tls.crt > {dir}/chain.crt", dir = directory), + format!("echo [{dir}] Creating keystore", dir = directory), format!("openssl pkcs12 -export -in {dir}/chain.crt -inkey {dir}/tls.key -out {dir}/keystore.p12 --passout file:{dir}/password", dir = directory), - "echo Cleaning up password".to_string(), + format!("echo [{dir}] Cleaning up password", dir = directory), format!("rm -f {dir}/password", dir = directory), - "echo chowning store directory".to_string(), + format!("echo [{dir}] Chowning store directory", dir = directory), format!("chown -R stackable:stackable {dir}", dir = directory), - "echo chmodding store directory".to_string(), + format!("echo [{dir}] Chmodding store directory", dir = directory), format!("chmod -R a=,u=rwX {dir}", dir = directory), ] } - -fn quorum_auth_properties(directory: &str, password: &str) -> BTreeMap { - vec![ - ("sslQuorum".to_string(), "true".to_string()), - ( - "serverCnxnFactory".to_string(), - "org.apache.zookeeper.server.NettyServerCnxnFactory".to_string(), - ), - ( - "ssl.quorum.keyStore.location".to_string(), - format!("{dir}/keystore.p12", dir = directory), - ), - ( - "ssl.quorum.keyStore.password".to_string(), - password.to_string(), - ), - ( - "ssl.quorum.trustStore.location".to_string(), - format!("{dir}/truststore.p12", dir = directory), - ), - ( - "ssl.quorum.trustStore.password".to_string(), - password.to_string(), - ), - ( - "ssl.quorum.hostnameVerification".to_string(), - "true".to_string(), - ), - ( - "authProvider.x509".to_string(), - "org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string(), - ), - ] - .into_iter() - .collect::>() -} - -fn client_auth_properties( - zk: &ZookeeperCluster, - directory: &str, - password: &str, -) -> BTreeMap { - vec![ - ("secureClientPort".to_string(), zk.client_port().to_string()), - ( - "serverCnxnFactory".to_string(), - "org.apache.zookeeper.server.NettyServerCnxnFactory".to_string(), - ), - ( - "ssl.keyStore.location".to_string(), - format!("{dir}/keystore.p12", dir = directory), - ), - ("ssl.keyStore.password".to_string(), password.to_string()), - ( - "ssl.trustStore.location".to_string(), - format!("{dir}/truststore.p12", dir = directory), - ), - ("ssl.trustStore.password".to_string(), password.to_string()), - ( - "authProvider.x509".to_string(), - "org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string(), - ), - ] - .into_iter() - .collect::>() -} diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs deleted file mode 100644 index db5385a8..00000000 --- a/rust/operator-binary/src/config.rs +++ /dev/null @@ -1,15 +0,0 @@ -use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperClusterSpec}; - -pub fn build_init_container_args(zk: &ZookeeperCluster) -> Vec { - //let mut args = vec![]; - let spec: &ZookeeperClusterSpec = &zk.spec; - - vec![ - "chown stackable:stackable /stackable/data", - "chmod a=,u=rwX /stackable/data", - "expr $MYID_OFFSET + $(echo $POD_NAME | sed 's/.*-//') > /stackable/data/myid", - ] - .join(" && "); - - vec!["test".to_string()] -} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 0af02d77..b0fdf881 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,5 +1,4 @@ mod auth_config; -mod config; mod discovery; mod utils; mod zk_controller; diff --git a/rust/operator-binary/src/zk_controller.rs b/rust/operator-binary/src/zk_controller.rs index 5fbcf754..eac4095d 100644 --- a/rust/operator-binary/src/zk_controller.rs +++ b/rust/operator-binary/src/zk_controller.rs @@ -8,9 +8,8 @@ use std::{ time::Duration, }; -use crate::config::build_init_container_args; use crate::{ - auth_config::{self, create_key_and_trust_store_cmd}, + auth_config::create_init_container_command_args, discovery::{self, build_discovery_configmaps}, APP_NAME, }; @@ -43,15 +42,13 @@ use stackable_operator::{ product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::RoleGroupRef, }; -use stackable_zookeeper_crd::{ZookeeperCluster, ZookeeperClusterStatus, ZookeeperRole}; +use stackable_zookeeper_crd::{ + ZookeeperCluster, ZookeeperClusterStatus, ZookeeperRole, CLIENT_TLS_DIR, QUORUM_TLS_DIR, + STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, +}; use strum::{EnumDiscriminants, IntoStaticStr}; const FIELD_MANAGER_SCOPE: &str = "zookeepercluster"; -const QUORUM_TLS_DIR: &str = "/stackable/tls/quorum"; -const CLIENT_TLS_DIR: &str = "/stackable/tls/client"; -// TODO: Generate? Currently this is written in the ConfigMap. -// It probably makes sense to add that after mounting the CM? -const TLS_STORE_SECRET: &str = "MyS3cr4T"; pub struct Ctx { pub client: stackable_operator::client::Client, @@ -403,10 +400,11 @@ fn build_server_rolegroup_statefulset( }) .collect::>(); - let mut container_prepare = ContainerBuilder::new("prepare") + let mut container_prepare = ContainerBuilder::new("prepare"); + container_prepare .image(&image) .command(vec!["sh".to_string(), "-c".to_string()]) - .args(build_init_container_args(zk)) + .args(vec![create_init_container_command_args(zk)]) .add_env_vars(env.clone()) .add_env_vars(vec![EnvVar { name: "POD_NAME".to_string(), @@ -419,19 +417,28 @@ fn build_server_rolegroup_statefulset( }), ..EnvVar::default() }]) - .add_volume_mount("data", "/stackable/data") - .add_volume_mount("quorum-tls", QUORUM_TLS_DIR) - .build(); + .add_volume_mount("data", STACKABLE_DATA_DIR); + + if zk.is_quorum_secure() { + container_prepare.add_volume_mount("quorum-tls", QUORUM_TLS_DIR); + } + if zk.is_client_secure() { + container_prepare.add_volume_mount("client-tls", CLIENT_TLS_DIR); + } + + let mut container_prepare = container_prepare.build(); container_prepare .security_context .get_or_insert_with(SecurityContext::default) .run_as_user = Some(0); - let container_zk = ContainerBuilder::new("zookeeper") + + let mut container_zk = ContainerBuilder::new("zookeeper"); + container_zk .image(image) .args(vec![ "bin/zkServer.sh".to_string(), "start-foreground".to_string(), - "/stackable/config/zoo.cfg".to_string(), + format!("{dir}/zoo.cfg", dir = STACKABLE_CONFIG_DIR), ]) .add_env_vars(env) // Only allow the global load balancing service to send traffic to pods that are members of the quorum @@ -456,10 +463,96 @@ fn build_server_rolegroup_statefulset( .add_container_port("zk-leader", 2888) .add_container_port("zk-election", 3888) .add_container_port("metrics", 9505) - .add_volume_mount("data", "/stackable/data") - .add_volume_mount("config", "/stackable/config") - .add_volume_mount("quorum-tls", QUORUM_TLS_DIR) - .build(); + .add_volume_mount("data", STACKABLE_DATA_DIR) + .add_volume_mount("config", STACKABLE_CONFIG_DIR); + + if zk.is_quorum_secure() { + container_zk.add_volume_mount("quorum-tls", QUORUM_TLS_DIR); + } + if zk.is_client_secure() { + container_zk.add_volume_mount("client-tls", CLIENT_TLS_DIR); + } + + let container_zk = container_zk.build(); + + let mut pod_builder = PodBuilder::new(); + pod_builder + .metadata_builder(|m| { + m.with_recommended_labels( + zk, + APP_NAME, + zk_version, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ) + }) + .add_init_container(container_prepare) + .add_container(container_zk) + .add_volume(Volume { + name: "config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: Some(rolegroup_ref.object_name()), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .security_context(PodSecurityContext { + fs_group: Some(1000), + ..PodSecurityContext::default() + }); + + if zk.is_quorum_secure() { + pod_builder.add_volume(Volume { + name: "quorum-tls".to_string(), + csi: Some(CSIVolumeSource { + driver: "secrets.stackable.tech".to_string(), + volume_attributes: Some( + vec![ + ( + "secrets.stackable.tech/class".to_string(), + "tls".to_string(), + ), + ( + "secrets.stackable.tech/scope".to_string(), + "node,pod".to_string(), + ), + ] + .into_iter() + .collect::>(), + ), + ..CSIVolumeSource::default() + }), + ..Volume::default() + }); + } + + if zk.is_client_secure() { + pod_builder.add_volume(Volume { + name: "client-tls".to_string(), + csi: Some(CSIVolumeSource { + driver: "secrets.stackable.tech".to_string(), + volume_attributes: Some( + vec![ + ( + "secrets.stackable.tech/class".to_string(), + "tls".to_string(), + ), + ( + "secrets.stackable.tech/scope".to_string(), + "node,pod".to_string(), + ), + ] + .into_iter() + .collect::>(), + ), + ..CSIVolumeSource::default() + }), + ..Volume::default() + }); + } + + let pod_builder = pod_builder.build_template(); + Ok(StatefulSet { metadata: ObjectMetaBuilder::new() .name_and_namespace(zk) @@ -491,53 +584,7 @@ fn build_server_rolegroup_statefulset( ..LabelSelector::default() }, service_name: rolegroup_ref.object_name(), - template: PodBuilder::new() - .metadata_builder(|m| { - m.with_recommended_labels( - zk, - APP_NAME, - zk_version, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - }) - .add_init_container(container_prepare) - .add_container(container_zk) - .add_volume(Volume { - name: "config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: Some(rolegroup_ref.object_name()), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .add_volume(Volume { - name: "quorum-tls".to_string(), - csi: Some(CSIVolumeSource { - driver: "secrets.stackable.tech".to_string(), - volume_attributes: Some( - vec![ - ( - "secrets.stackable.tech/class".to_string(), - "tls".to_string(), - ), - ( - "secrets.stackable.tech/scope".to_string(), - "node,pod".to_string(), - ), - ] - .into_iter() - .collect::>(), - ), - ..CSIVolumeSource::default() - }), - ..Volume::default() - }) - .security_context(PodSecurityContext { - fs_group: Some(1000), - ..PodSecurityContext::default() - }) - .build_template(), + template: pod_builder, volume_claim_templates: Some(vec![PersistentVolumeClaim { metadata: ObjectMeta { name: Some("data".to_string()), From 3a07444625ae25c769960c4f516c40fa056ab2cb Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 5 May 2022 16:46:56 +0200 Subject: [PATCH 05/27] quorum and client auth working --- Cargo.lock | 340 +++++++++++---------- deploy/crd/zookeepercluster.crd.yaml | 3 + examples/simple-zookeeper-tls-cluster.yaml | 24 +- rust/crd/Cargo.toml | 3 +- rust/crd/src/lib.rs | 124 +++++--- rust/operator-binary/Cargo.toml | 6 +- rust/operator-binary/src/auth_config.rs | 12 +- rust/operator-binary/src/zk_controller.rs | 115 +++++-- 8 files changed, 381 insertions(+), 246 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 391df40b..a5f034be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,9 +48,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" [[package]] name = "async-trait" @@ -93,9 +93,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" +checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61" dependencies = [ "addr2line", "cc", @@ -174,11 +174,11 @@ checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "cargo-lock" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb04b88bd5b2036e30704f95c6ee16f3b5ca3b4ca307da2889d9006648e5c88" +checksum = "6c408da54db4c50d4693f7e649c299bc9de9c23ead86249e5368830bb32a734b" dependencies = [ - "semver 1.0.7", + "semver 1.0.9", "serde", "toml", "url", @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.12" +version = "3.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db" +checksum = "85a35a599b11c089a7f49105658d089b8f2cf0882993c17daf6de15285c2c35d" dependencies = [ "atty", "bitflags", @@ -242,7 +242,7 @@ version = "3.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro-error", "proc-macro2", "quote", @@ -251,9 +251,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669" +checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" dependencies = [ "os_str_bytes", ] @@ -269,9 +269,9 @@ dependencies = [ [[package]] name = "const_format" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22bc6cd49b0ec407b680c3e380182b6ac63b73991cb7602de350352fc309b614" +checksum = "0936ffe6d0c8d6a51b3b0a73b2acbe925d786f346cf45bfddc8341d79fb7dc8a" dependencies = [ "const_format_proc_macros", ] @@ -373,9 +373,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ "darling_core", "darling_macro", @@ -383,9 +383,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", @@ -397,9 +397,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core", "quote", @@ -706,9 +706,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", "libc", @@ -740,15 +740,6 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.0" @@ -766,9 +757,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" dependencies = [ "bytes 1.1.0", "fnv", @@ -794,9 +785,9 @@ checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" [[package]] name = "httparse" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" [[package]] name = "httpdate" @@ -806,9 +797,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.17" +version = "0.14.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043f0e083e9901b6cc658a77d1eb86f4fc650bbb977a4337dd63192826aa85dd" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" dependencies = [ "bytes 1.1.0", "futures-channel", @@ -821,7 +812,7 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio 1.18.0", + "tokio 1.18.1", "tower-service", "tracing", "want", @@ -840,7 +831,7 @@ dependencies = [ "openssl", "openssl-sys", "parking_lot 0.12.0", - "tokio 1.18.0", + "tokio 1.18.1", "tokio-openssl", "tower-layer", ] @@ -853,7 +844,7 @@ checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ "hyper", "pin-project-lite", - "tokio 1.18.0", + "tokio 1.18.1", "tokio-io-timeout", ] @@ -866,7 +857,7 @@ dependencies = [ "bytes 1.1.0", "hyper", "native-tls", - "tokio 1.18.0", + "tokio 1.18.1", "tokio-native-tls", ] @@ -889,9 +880,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", "hashbrown", @@ -1045,7 +1036,7 @@ dependencies = [ "serde_json", "serde_yaml", "thiserror", - "tokio 1.18.0", + "tokio 1.18.1", "tokio-native-tls", "tokio-util", "tower", @@ -1103,7 +1094,7 @@ dependencies = [ "serde_json", "smallvec 1.8.0", "thiserror", - "tokio 1.18.0", + "tokio 1.18.1", "tokio-util", "tracing", ] @@ -1116,9 +1107,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.121" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" +checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" [[package]] name = "libgit2-sys" @@ -1134,9 +1125,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e" dependencies = [ "cc", "libc", @@ -1170,18 +1161,19 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", ] @@ -1209,9 +1201,9 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -1224,12 +1216,11 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" dependencies = [ "adler", - "autocfg", ] [[package]] @@ -1299,9 +1290,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", @@ -1337,9 +1328,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -1347,9 +1338,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -1366,9 +1357,9 @@ dependencies = [ [[package]] name = "object" -version = "0.27.1" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456" dependencies = [ "memchr", ] @@ -1381,18 +1372,30 @@ checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" [[package]] name = "openssl" -version = "0.10.38" +version = "0.10.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" dependencies = [ "bitflags", "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1401,9 +1404,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" dependencies = [ "autocfg", "cc", @@ -1429,7 +1432,7 @@ dependencies = [ "pin-project", "rand", "thiserror", - "tokio 1.18.0", + "tokio 1.18.1", "tokio-stream", ] @@ -1445,7 +1448,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "thiserror", "thrift", - "tokio 1.18.0", + "tokio 1.18.1", ] [[package]] @@ -1498,8 +1501,8 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ - "lock_api 0.4.6", - "parking_lot_core 0.9.1", + "lock_api 0.4.7", + "parking_lot_core 0.9.3", ] [[package]] @@ -1519,13 +1522,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.11", + "redox_syscall 0.2.13", "smallvec 1.8.0", "windows-sys", ] @@ -1567,9 +1570,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -1579,9 +1582,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "ppv-lite86" @@ -1615,9 +1618,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" dependencies = [ "unicode-xid", ] @@ -1630,7 +1633,7 @@ dependencies = [ "fancy-regex", "java-properties", "schemars", - "semver 1.0.7", + "semver 1.0.9", "serde", "serde_json", "serde_yaml", @@ -1640,9 +1643,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -1685,21 +1688,21 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "redox_syscall 0.2.11", + "redox_syscall 0.2.13", "thiserror", ] @@ -1849,9 +1852,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" dependencies = [ "serde", ] @@ -1864,9 +1867,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] @@ -1883,9 +1886,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -1905,9 +1908,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "indexmap", "itoa", @@ -1917,9 +1920,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0" +checksum = "707d15895415db6628332b737c838b88c598522e4dc70647e59b72312924aebc" dependencies = [ "indexmap", "ryu", @@ -1947,9 +1950,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "slog" @@ -1974,9 +1977,9 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "snafu" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eba135d2c579aa65364522eb78590cdf703176ef71ad4c32b00f58f7afb2df5" +checksum = "5177903bf45656592d9eb5c0e22f408fc023aae51dbe2088889b71633ba451f2" dependencies = [ "doc-comment", "snafu-derive", @@ -1984,11 +1987,11 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a7fe9b0669ef117c5cabc5549638528f36771f058ff977d7689deb517833a75" +checksum = "410b26ed97440d90ced3e2488c868d56a86e2064f5d7d6f417909b286afe25e5" dependencies = [ - "heck 0.3.3", + "heck", "proc-macro2", "quote", "syn", @@ -2006,7 +2009,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.17.0" +version = "0.19.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.19.0#f8f2d5527b3463cc40f3d851172af7ae81db75c1" dependencies = [ "backoff", "chrono", @@ -2028,14 +2032,26 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "stackable-operator-derive", "strum", "thiserror", - "tokio 1.18.0", + "tokio 1.18.1", "tracing", "tracing-opentelemetry", "tracing-subscriber", ] +[[package]] +name = "stackable-operator-derive" +version = "0.17.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.19.0#f8f2d5527b3463cc40f3d851172af7ae81db75c1" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "stackable-zookeeper-crd" version = "0.10.0-nightly" @@ -2058,7 +2074,7 @@ dependencies = [ "fnv", "futures 0.3.21", "pin-project", - "semver 1.0.7", + "semver 1.0.9", "serde", "serde_json", "serde_yaml", @@ -2067,7 +2083,7 @@ dependencies = [ "stackable-zookeeper-crd", "strum", "tokio 0.1.22", - "tokio 1.18.0", + "tokio 1.18.1", "tokio-executor", "tokio-zookeeper", "tracing", @@ -2094,7 +2110,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro2", "quote", "rustversion", @@ -2103,9 +2119,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" dependencies = [ "proc-macro2", "quote", @@ -2133,7 +2149,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand", "libc", - "redox_syscall 0.2.11", + "redox_syscall 0.2.13", "remove_dir_all", "winapi 0.3.9", ] @@ -2155,18 +2171,18 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -2216,9 +2232,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -2255,9 +2271,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f48b6d60512a392e34dbf7fd456249fd2de3c83669ab642e021903f4015185b" +checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc" dependencies = [ "bytes 1.1.0", "libc", @@ -2333,7 +2349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ "pin-project-lite", - "tokio 1.18.0", + "tokio 1.18.1", ] [[package]] @@ -2354,7 +2370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.18.0", + "tokio 1.18.1", ] [[package]] @@ -2366,7 +2382,7 @@ dependencies = [ "futures-util", "openssl", "openssl-sys", - "tokio 1.18.0", + "tokio 1.18.1", ] [[package]] @@ -2396,7 +2412,7 @@ checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" dependencies = [ "futures-core", "pin-project-lite", - "tokio 1.18.0", + "tokio 1.18.1", ] [[package]] @@ -2487,17 +2503,17 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" dependencies = [ "bytes 1.1.0", "futures-core", "futures-sink", - "log", "pin-project-lite", "slab", - "tokio 1.18.0", + "tokio 1.18.1", + "tracing", ] [[package]] @@ -2516,9 +2532,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] @@ -2533,7 +2549,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio 1.18.0", + "tokio 1.18.1", "tokio-util", "tower-layer", "tower-service", @@ -2587,9 +2603,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" dependencies = [ "proc-macro2", "quote", @@ -2598,9 +2614,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", "valuable", @@ -2608,9 +2624,9 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", "log", @@ -2665,9 +2681,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-normalization" @@ -2678,17 +2694,11 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" - [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "url" @@ -2841,9 +2851,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -2854,33 +2864,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "ws2_32-sys" @@ -2909,6 +2919,6 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb5728b8afd3f280a869ce1d4c554ffaed35f45c231fc41bfbd0381bef50317" +checksum = "94693807d016b2f2d2e14420eb3bfcca689311ff775dcf113d74ea624b7cdf07" diff --git a/deploy/crd/zookeepercluster.crd.yaml b/deploy/crd/zookeepercluster.crd.yaml index 070b2cf4..a514ef7a 100644 --- a/deploy/crd/zookeepercluster.crd.yaml +++ b/deploy/crd/zookeepercluster.crd.yaml @@ -27,6 +27,7 @@ spec: nullable: true properties: clientAuthentication: + description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" nullable: true properties: authenticationClass: @@ -35,8 +36,10 @@ spec: - authenticationClass type: object quorumTlsSecretClass: + description: "Only affects quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" type: string tls: + description: "Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: \"tls\".to_string() }." nullable: true properties: secretClass: diff --git a/examples/simple-zookeeper-tls-cluster.yaml b/examples/simple-zookeeper-tls-cluster.yaml index 884fc9c8..e8285656 100644 --- a/examples/simple-zookeeper-tls-cluster.yaml +++ b/examples/simple-zookeeper-tls-cluster.yaml @@ -9,7 +9,7 @@ spec: tls: secretClass: tls clientAuthentication: - authenticationClass: zookeeper-client-tls + authenticationClass: zk-client-tls quorumTlsSecretClass: tls servers: roleGroups: @@ -19,14 +19,24 @@ spec: apiVersion: authentication.stackable.tech/v1alpha1 kind: AuthenticationClass metadata: - name: zookeeper-client-tls + name: zk-client-tls spec: provider: tls: - verification: - server: - caCert: - secretClass: tls + clientCertSecretClass: zk-client-auth-secret +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: zk-client-auth-secret +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-zk-client-ca + namespace: default + autoGenerate: true --- apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperZnode @@ -36,4 +46,4 @@ spec: clusterRef: name: simple-zk # Optional when ZookeeperZnode is in the same Namespace as the ZookeeperCluster - # namespace: default \ No newline at end of file + # namespace: default diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index 14f94c72..8ac4091c 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -11,6 +11,5 @@ repository = "https://github.com/stackabletech/zookeeper-operator" serde = "1.0.136" serde_json = "1.0.79" snafu = "0.7.0" -#stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.18.0" } -stackable-operator = { path = "../../../operator-rs"} +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.19.0" } strum = { version = "0.24.0", features = ["derive"] } diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index f6c2b108..4fced609 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -21,6 +21,8 @@ pub const CLIENT_TLS_DIR: &str = "/stackable/tls/client"; // It probably makes sense to add that after mounting the CM? pub const TLS_STORE_SECRET: &str = "My_5t4ck4ble_53cr4T"; +const DEFAULT_SECRET_CLASS: &str = "tls"; + /// A cluster of ZooKeeper nodes #[derive(Clone, CustomResource, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[kube( @@ -58,14 +60,38 @@ pub struct GlobalZookeeperConfig { pub tls_config: Option, } -#[derive(Clone, Default, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct TlsConfig { + /// Only affects client connections. This setting controls: + /// - If TLS encryption is used at all + /// - Which cert the servers should use to authenticate themselves against the client + /// Defaults to `TlsSecretClass` { secret_class: "tls".to_string() }. pub tls: Option, + /// Only affects client connections. This setting controls: + /// - If clients need to authenticate themselves against the server via TLS + /// - Which ca.crt to use when validating the provided client certs + /// Defaults to `None` pub client_authentication: Option, + /// Only affects quorum communication. Use mutual verification between Zookeeper Nodes + /// (mandatory). This setting controls: + /// - Which cert the servers should use to authenticate themselves against other servers + /// - Which ca.crt to use when validating the other server pub quorum_tls_secret_class: String, } +impl Default for TlsConfig { + fn default() -> Self { + Self { + tls: Some(TlsSecretClass { + secret_class: DEFAULT_SECRET_CLASS.to_string(), + }), + client_authentication: None, + quorum_tls_secret_class: DEFAULT_SECRET_CLASS.to_string(), + } + } +} + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct TlsSecretClass { @@ -75,7 +101,7 @@ pub struct TlsSecretClass { impl Default for TlsSecretClass { fn default() -> Self { Self { - secret_class: "tls".to_string(), + secret_class: DEFAULT_SECRET_CLASS.to_string(), } } } @@ -107,6 +133,8 @@ impl ZookeeperConfig { // Quorum TLS pub const SSL_QUORUM: &'static str = "sslQuorum"; + pub const SSL_QUORUM_CLIENT_AUTH: &'static str = "ssl.quorum.clientAuth"; + pub const SSL_QUORUM_HOST_NAME_VERIFICATION: &'static str = "ssl.quorum.hostnameVerification"; pub const SSL_QUORUM_KEY_STORE_LOCATION: &'static str = "ssl.quorum.keyStore.location"; pub const SSL_QUORUM_KEY_STORE_PASSWORD: &'static str = "ssl.quorum.keyStore.password"; pub const SSL_QUORUM_TRUST_STORE_LOCATION: &'static str = "ssl.quorum.trustStore.location"; @@ -118,6 +146,7 @@ impl ZookeeperConfig { pub const SSL_TRUST_STORE_LOCATION: &'static str = "ssl.trustStore.location"; pub const SSL_TRUST_STORE_PASSWORD: &'static str = "ssl.trustStore.password"; pub const SSL_AUTH_PROVIDER_X509: &'static str = "authProvider.x509"; + pub const SSL_CLIENT_AUTH: &'static str = "ssl.clientAuth"; // Common TLS pub const SERVER_CNXN_FACTORY: &'static str = "serverCnxnFactory"; @@ -175,29 +204,36 @@ impl Configuration for ZookeeperConfig { ); // Quorum TLS - if resource.is_quorum_secure() { - result.insert(Self::SSL_QUORUM.to_string(), Some("true".to_string())); - result.insert( - Self::SERVER_CNXN_FACTORY.to_string(), - Some("org.apache.zookeeper.server.NettyServerCnxnFactory".to_string()), - ); - result.insert( - Self::SSL_QUORUM_KEY_STORE_LOCATION.to_string(), - Some(format!("{dir}/keystore.p12", dir = QUORUM_TLS_DIR)), - ); - result.insert( - Self::SSL_QUORUM_KEY_STORE_PASSWORD.to_string(), - Some(TLS_STORE_SECRET.to_string()), - ); - result.insert( - Self::SSL_QUORUM_TRUST_STORE_LOCATION.to_string(), - Some(format!("{dir}/truststore.p12", dir = QUORUM_TLS_DIR)), - ); - result.insert( - Self::SSL_QUORUM_TRUST_STORE_PASSWORD.to_string(), - Some(TLS_STORE_SECRET.to_string()), - ); - } + result.insert(Self::SSL_QUORUM.to_string(), Some("true".to_string())); + result.insert( + Self::SSL_QUORUM_HOST_NAME_VERIFICATION.to_string(), + Some("true".to_string()), + ); + result.insert( + Self::SSL_QUORUM_CLIENT_AUTH.to_string(), + Some("need".to_string()), + ); + result.insert( + Self::SERVER_CNXN_FACTORY.to_string(), + Some("org.apache.zookeeper.server.NettyServerCnxnFactory".to_string()), + ); + result.insert( + Self::SSL_QUORUM_KEY_STORE_LOCATION.to_string(), + Some(format!("{dir}/keystore.p12", dir = QUORUM_TLS_DIR)), + ); + result.insert( + Self::SSL_QUORUM_KEY_STORE_PASSWORD.to_string(), + Some(TLS_STORE_SECRET.to_string()), + ); + result.insert( + Self::SSL_QUORUM_TRUST_STORE_LOCATION.to_string(), + Some(format!("{dir}/truststore.p12", dir = QUORUM_TLS_DIR)), + ); + result.insert( + Self::SSL_QUORUM_TRUST_STORE_PASSWORD.to_string(), + Some(TLS_STORE_SECRET.to_string()), + ); + // Client TLS if resource.is_client_secure() { // We set only the clientPort and portUnification here because otherwise there is a port bind exception @@ -215,7 +251,7 @@ impl Configuration for ZookeeperConfig { // Some(SECURE_CLIENT_PORT.to_string()), // ); - // 2) Setting only secureClientPort will result in the above mentioned bind exception + // 2) Setting only secureClientPort will result in the above mentioned bind exception. // The NettyFactory tries to bind multiple times on the secureClientPort. // result.insert( // Self::SECURE_CLIENT_PORT.to_string(), @@ -258,6 +294,11 @@ impl Configuration for ZookeeperConfig { Self::SSL_AUTH_PROVIDER_X509.to_string(), Some("org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string()), ); + + // Check if we need to enable authentication + if resource.client_tls_authentication_class().is_some() { + result.insert(Self::SSL_CLIENT_AUTH.to_string(), Some("need".to_string())); + } } else { result.insert( Self::CLIENT_PORT.to_string(), @@ -343,52 +384,53 @@ impl ZookeeperCluster { } pub fn client_port(&self) -> u16 { - if self.client_auth().is_some() { + if self.is_client_secure() { SECURE_CLIENT_PORT } else { CLIENT_PORT } } - pub fn is_quorum_secure(&self) -> bool { + /// Returns the secret class for client connection encryption. Defaults to `tls`. + pub fn client_tls_secret_class(&self) -> String { let spec: &ZookeeperClusterSpec = &self.spec; spec.config .as_ref() .and_then(|c| c.tls_config.as_ref()) - .is_some() + .and_then(|tls| tls.tls.clone()) + .unwrap_or_default() + .secret_class } + /// Checks if we should use TLS to encrypt client connections. pub fn is_client_secure(&self) -> bool { - let spec: &ZookeeperClusterSpec = &self.spec; - spec.config - .as_ref() - .and_then(|c| c.tls_config.as_ref()) - .and_then(|tls| tls.client_authentication.as_ref()) - .is_some() - } - - pub fn tls_secret_class(&self) -> Option<&TlsSecretClass> { let spec: &ZookeeperClusterSpec = &self.spec; spec.config .as_ref() .and_then(|c| c.tls_config.as_ref()) .and_then(|tls| tls.tls.as_ref()) + .is_some() } - pub fn client_auth(&self) -> Option<&ClientAuthenticationClass> { + /// Returns the authentication class used for client authentication + pub fn client_tls_authentication_class(&self) -> Option { let spec: &ZookeeperClusterSpec = &self.spec; spec.config .as_ref() .and_then(|c| c.tls_config.as_ref()) .and_then(|tls| tls.client_authentication.as_ref()) + .map(|auth| auth.authentication_class.clone()) } - pub fn quorum_tls_secret_class(&self) -> Option<&String> { + /// Returns the secret class for internal server encryption + pub fn quorum_tls_secret_class(&self) -> String { let spec: &ZookeeperClusterSpec = &self.spec; spec.config .as_ref() .and_then(|c| c.tls_config.as_ref()) - .map(|tls| &tls.quorum_tls_secret_class) + .map(|tls| tls.quorum_tls_secret_class.as_ref()) + .unwrap_or(DEFAULT_SECRET_CLASS) + .to_string() } } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 16fbff5b..454ae76c 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -25,12 +25,10 @@ tokio-executor = "0.1.10" tokio-zookeeper = "0.1.3" tracing = "0.1.33" pin-project = "1.0.10" -#stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.18.0" } -stackable-operator = { path = "../../../operator-rs"} +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.19.0" } stackable-zookeeper-crd = { path = "../crd" } [build-dependencies] built = { version = "0.5.1", features = ["chrono", "git2"] } -#stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.18.0" } -stackable-operator = { path = "../../../operator-rs"} +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.19.0" } stackable-zookeeper-crd = { path = "../crd" } diff --git a/rust/operator-binary/src/auth_config.rs b/rust/operator-binary/src/auth_config.rs index 63732c42..4113c998 100644 --- a/rust/operator-binary/src/auth_config.rs +++ b/rust/operator-binary/src/auth_config.rs @@ -4,13 +4,11 @@ use stackable_zookeeper_crd::{ pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { let mut args = vec![]; - // TODO: we need to check if another secret / authentication class is used - if zk.is_quorum_secure() { - args.extend(create_key_and_trust_store_cmd( - QUORUM_TLS_DIR, - TLS_STORE_SECRET, - )); - } + + args.extend(create_key_and_trust_store_cmd( + QUORUM_TLS_DIR, + TLS_STORE_SECRET, + )); if zk.is_client_secure() { args.extend(create_key_and_trust_store_cmd( diff --git a/rust/operator-binary/src/zk_controller.rs b/rust/operator-binary/src/zk_controller.rs index eac4095d..a43fd833 100644 --- a/rust/operator-binary/src/zk_controller.rs +++ b/rust/operator-binary/src/zk_controller.rs @@ -11,13 +11,16 @@ use std::{ use crate::{ auth_config::create_init_container_command_args, discovery::{self, build_discovery_configmaps}, - APP_NAME, + ObjectRef, APP_NAME, }; use fnv::FnvHasher; use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::commons::authentication::AuthenticationClassProvider; use stackable_operator::k8s_openapi::api::core::v1::CSIVolumeSource; +use stackable_operator::kube::api::DynamicObject; use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, + commons::authentication::AuthenticationClass, k8s_openapi::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, @@ -122,6 +125,21 @@ pub enum Error { ApplyStatus { source: stackable_operator::error::Error, }, + #[snafu(display("failed to retrieve {}", authentication_class))] + AuthenticationClassRetrieval { + source: stackable_operator::error::Error, + authentication_class: ObjectRef, + }, + #[snafu(display( + "failed to use authentication mechanism {} - supported methods: {:?}", + method, + supported + ))] + AuthenticationMethodNotSupported { + authentication_class: ObjectRef, + supported: Vec, + method: String, + }, } type Result = std::result::Result; @@ -129,6 +147,35 @@ impl ReconcilerError for Error { fn category(&self) -> &'static str { ErrorDiscriminants::from(self).into() } + fn secondary_object(&self) -> Option> { + match self { + Error::ObjectHasNoNamespace => None, + Error::ObjectHasNoVersion => None, + Error::NoServerRole => None, + Error::GlobalServiceNameNotFound => None, + Error::RoleGroupServiceNameNotFound { .. } => None, + Error::ApplyRoleService { .. } => None, + Error::ApplyRoleGroupService { .. } => None, + Error::BuildRoleGroupConfig { .. } => None, + Error::ApplyRoleGroupConfig { .. } => None, + Error::ApplyRoleGroupStatefulSet { .. } => None, + Error::GenerateProductConfig { .. } => None, + Error::InvalidProductConfig { .. } => None, + Error::SerializeZooCfg { .. } => None, + Error::ObjectMissingMetadataForOwnerRef { .. } => None, + Error::BuildDiscoveryConfig { .. } => None, + Error::ApplyDiscoveryConfig { .. } => None, + Error::ApplyStatus { .. } => None, + Error::AuthenticationClassRetrieval { + authentication_class, + .. + } => Some(authentication_class.clone().erase()), + Error::AuthenticationMethodNotSupported { + authentication_class, + .. + } => Some(authentication_class.clone().erase()), + } + } } const PROPERTIES_FILE: &str = "zoo.cfg"; @@ -168,6 +215,20 @@ pub async fn reconcile_zk( .map(Cow::Borrowed) .unwrap_or_default(); + let client_authentication_class = if let Some(auth_class) = zk.client_tls_authentication_class() + { + Some( + client + .get::(&auth_class, None) // AuthenticationClass has ClusterScope + .await + .context(AuthenticationClassRetrievalSnafu { + authentication_class: ObjectRef::::new(&auth_class), + })?, + ) + } else { + None + }; + let server_role_service = build_server_role_service(&zk)?; let server_role_service = client .apply_patch( @@ -182,7 +243,12 @@ pub async fn reconcile_zk( let rg_service = build_server_rolegroup_service(&rolegroup, &zk)?; let rg_configmap = build_server_rolegroup_config_map(&rolegroup, &zk, rolegroup_config)?; - let rg_statefulset = build_server_rolegroup_statefulset(&rolegroup, &zk, rolegroup_config)?; + let rg_statefulset = build_server_rolegroup_statefulset( + &rolegroup, + &zk, + rolegroup_config, + client_authentication_class.as_ref(), + )?; client .apply_patch(FIELD_MANAGER_SCOPE, &rg_service, &rg_service) .await @@ -376,6 +442,7 @@ fn build_server_rolegroup_statefulset( rolegroup_ref: &RoleGroupRef, zk: &ZookeeperCluster, server_config: &HashMap>, + client_authentication_class: Option<&AuthenticationClass>, ) -> Result { let rolegroup = zk .spec @@ -417,11 +484,9 @@ fn build_server_rolegroup_statefulset( }), ..EnvVar::default() }]) - .add_volume_mount("data", STACKABLE_DATA_DIR); + .add_volume_mount("data", STACKABLE_DATA_DIR) + .add_volume_mount("quorum-tls", QUORUM_TLS_DIR); - if zk.is_quorum_secure() { - container_prepare.add_volume_mount("quorum-tls", QUORUM_TLS_DIR); - } if zk.is_client_secure() { container_prepare.add_volume_mount("client-tls", CLIENT_TLS_DIR); } @@ -464,11 +529,9 @@ fn build_server_rolegroup_statefulset( .add_container_port("zk-election", 3888) .add_container_port("metrics", 9505) .add_volume_mount("data", STACKABLE_DATA_DIR) - .add_volume_mount("config", STACKABLE_CONFIG_DIR); + .add_volume_mount("config", STACKABLE_CONFIG_DIR) + .add_volume_mount("quorum-tls", QUORUM_TLS_DIR); - if zk.is_quorum_secure() { - container_zk.add_volume_mount("quorum-tls", QUORUM_TLS_DIR); - } if zk.is_client_secure() { container_zk.add_volume_mount("client-tls", CLIENT_TLS_DIR); } @@ -496,13 +559,7 @@ fn build_server_rolegroup_statefulset( }), ..Volume::default() }) - .security_context(PodSecurityContext { - fs_group: Some(1000), - ..PodSecurityContext::default() - }); - - if zk.is_quorum_secure() { - pod_builder.add_volume(Volume { + .add_volume(Volume { name: "quorum-tls".to_string(), csi: Some(CSIVolumeSource { driver: "secrets.stackable.tech".to_string(), @@ -510,7 +567,7 @@ fn build_server_rolegroup_statefulset( vec![ ( "secrets.stackable.tech/class".to_string(), - "tls".to_string(), + zk.quorum_tls_secret_class(), ), ( "secrets.stackable.tech/scope".to_string(), @@ -523,10 +580,28 @@ fn build_server_rolegroup_statefulset( ..CSIVolumeSource::default() }), ..Volume::default() + }) + .security_context(PodSecurityContext { + fs_group: Some(1000), + ..PodSecurityContext::default() }); - } if zk.is_client_secure() { + let secret_class = if let Some(auth_class) = client_authentication_class { + match &auth_class.spec.provider { + AuthenticationClassProvider::Tls(tls) => tls.client_cert_secret_class.clone(), + _ => { + return Err(Error::AuthenticationMethodNotSupported { + authentication_class: ObjectRef::from_obj(auth_class), + supported: vec!["tls".to_string()], + method: auth_class.spec.provider.to_string(), + }) + } + } + } else { + None + }; + pod_builder.add_volume(Volume { name: "client-tls".to_string(), csi: Some(CSIVolumeSource { @@ -535,7 +610,7 @@ fn build_server_rolegroup_statefulset( vec![ ( "secrets.stackable.tech/class".to_string(), - "tls".to_string(), + secret_class.unwrap_or_else(|| zk.client_tls_secret_class()), ), ( "secrets.stackable.tech/scope".to_string(), From 92f6a222612fb3bcab0e9a83f845b89486b31669 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 5 May 2022 17:12:05 +0200 Subject: [PATCH 06/27] extended product-config properties --- deploy/config-spec/properties.yaml | 87 ++++++++++++++++++++++++++++-- rust/crd/src/lib.rs | 23 ++++---- 2 files changed, 94 insertions(+), 16 deletions(-) diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index a18e87b1..c78894d8 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -159,12 +159,52 @@ properties: type: "bool" defaultValues: - value: "false" + recommendedValues: + - value: "true" roles: - name: "server" required: false asOfVersion: "3.5.5" description: "Enables encrypted quorum communication." + - property: &sslQuorumClientAuth + propertyNames: + - name: "ssl.quorum.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from clients." + + - property: &sslQuorumHostNameVerification + propertyNames: + - name: "ssl.quorum.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in quorum TLS negotiation process. Disabling it only recommended for testing purposes." + - property: &sslQuorumKeyStoreLocation propertyNames: - name: "ssl.quorum.keyStore.location" @@ -230,6 +270,44 @@ properties: ########################## # Client TLS ########################## + - property: &sslClientAuth + propertyNames: + - name: "ssl.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from clients." + + - property: &sslHostNameVerification + propertyNames: + - name: "ssl.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in client TLS negotiation process. Disabling it only recommended for testing purposes." + - property: &sslKeyStoreLocation propertyNames: - name: "ssl.keyStore.location" @@ -291,7 +369,9 @@ properties: required: false asOfVersion: "3.5.5" description: "The password to unlock the client truststore." - + ########################## + # Common TLS + ########################## - property: &authProviderX509 propertyNames: - name: "authProvider.x509" @@ -304,11 +384,8 @@ properties: - name: "server" required: false asOfVersion: "0.0.0" - description: "Used for secure client authentication.." + description: "Used for secure client authentication." - ########################## - # Common TLS - ########################## - property: &serverCnxnFactory propertyNames: - name: "serverCnxnFactory" diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 4fced609..7967660d 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -141,13 +141,14 @@ impl ZookeeperConfig { pub const SSL_QUORUM_TRUST_STORE_PASSWORD: &'static str = "ssl.quorum.trustStore.password"; // Client TLS pub const SECURE_CLIENT_PORT: &'static str = "secureClientPort"; + pub const SSL_CLIENT_AUTH: &'static str = "ssl.clientAuth"; + pub const SSL_HOST_NAME_VERIFICATION: &'static str = "ssl.hostnameVerification"; pub const SSL_KEY_STORE_LOCATION: &'static str = "ssl.keyStore.location"; pub const SSL_KEY_STORE_PASSWORD: &'static str = "ssl.keyStore.password"; pub const SSL_TRUST_STORE_LOCATION: &'static str = "ssl.trustStore.location"; pub const SSL_TRUST_STORE_PASSWORD: &'static str = "ssl.trustStore.password"; - pub const SSL_AUTH_PROVIDER_X509: &'static str = "authProvider.x509"; - pub const SSL_CLIENT_AUTH: &'static str = "ssl.clientAuth"; // Common TLS + pub const SSL_AUTH_PROVIDER_X509: &'static str = "authProvider.x509"; pub const SERVER_CNXN_FACTORY: &'static str = "serverCnxnFactory"; fn myid_offset(&self) -> u16 { @@ -217,6 +218,10 @@ impl Configuration for ZookeeperConfig { Self::SERVER_CNXN_FACTORY.to_string(), Some("org.apache.zookeeper.server.NettyServerCnxnFactory".to_string()), ); + result.insert( + Self::SSL_AUTH_PROVIDER_X509.to_string(), + Some("org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string()), + ); result.insert( Self::SSL_QUORUM_KEY_STORE_LOCATION.to_string(), Some(format!("{dir}/keystore.p12", dir = QUORUM_TLS_DIR)), @@ -265,15 +270,16 @@ impl Configuration for ZookeeperConfig { Some(resource.client_port().to_string()), ); result.insert( - "client.portUnification".to_string(), + Self::SSL_HOST_NAME_VERIFICATION.to_string(), Some("true".to_string()), ); - // END TICKET result.insert( - Self::SERVER_CNXN_FACTORY.to_string(), - Some("org.apache.zookeeper.server.NettyServerCnxnFactory".to_string()), + "client.portUnification".to_string(), + Some("true".to_string()), ); + // END TICKET + result.insert( Self::SSL_KEY_STORE_LOCATION.to_string(), Some(format!("{dir}/keystore.p12", dir = CLIENT_TLS_DIR)), @@ -290,11 +296,6 @@ impl Configuration for ZookeeperConfig { Self::SSL_TRUST_STORE_PASSWORD.to_string(), Some(TLS_STORE_SECRET.to_string()), ); - result.insert( - Self::SSL_AUTH_PROVIDER_X509.to_string(), - Some("org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string()), - ); - // Check if we need to enable authentication if resource.client_tls_authentication_class().is_some() { result.insert(Self::SSL_CLIENT_AUTH.to_string(), Some("need".to_string())); From 748fe061d596c2cc8ef189f39c47e681591e0787 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 6 May 2022 12:56:21 +0200 Subject: [PATCH 07/27] adapted docs --- CHANGELOG.md | 4 +- .../example-cluster-tls-authentication.yaml | 40 +++++++++ .../example-cluster-tls-encryption.yaml | 15 ++++ .../example-secret-operator-tls-secret.yaml | 13 +++ docs/modules/ROOT/pages/usage.adoc | 90 +++++++++++++------ 5 files changed, 134 insertions(+), 28 deletions(-) create mode 100644 docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml create mode 100644 docs/modules/ROOT/examples/example-cluster-tls-encryption.yaml create mode 100644 docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index cd732110..29bdd959 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,16 +11,18 @@ All notable changes to this project will be documented in this file. a single namespace to watch ([#434]). - Support for ZooKeeper 3.8.0 added ([#464]). - Integration tests for all supported ZooKeeper versions added ([#464]). +- TLS encryption and authentication support for quorum and client ([#xxx]). ### Changed -- Operator-rs: 0.10.0 -> 0.15.0 ([#408], [#431], [#434], [#454]). +- Operator-rs: 0.10.0 -> 0.19.0 ([#408], [#431], [#434], [#454], [#xxx]). [#408]: https://github.com/stackabletech/zookeeper-operator/pull/408 [#431]: https://github.com/stackabletech/zookeeper-operator/pull/431 [#434]: https://github.com/stackabletech/zookeeper-operator/pull/434 [#454]: https://github.com/stackabletech/zookeeper-operator/pull/454 [#464]: https://github.com/stackabletech/zookeeper-operator/pull/464 +[#xxx]: https://github.com/stackabletech/zookeeper-operator/pull/xxx ## [0.9.0] - 2022-02-14 diff --git a/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml b/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml new file mode 100644 index 00000000..cc0a6a2e --- /dev/null +++ b/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml @@ -0,0 +1,40 @@ + +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: simple-zk +spec: + version: 3.8.0 + config: + tls: + secretClass: tls + clientAuthentication: + authenticationClass: zk-client-tls # <1> + quorumTlsSecretClass: tls + servers: + roleGroups: + default: + replicas: 3 +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: zk-client-tls # <2> +spec: + provider: + tls: + clientCertSecretClass: zk-client-auth-secret # <3> +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: zk-client-auth-secret # <4> +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-zk-client-ca + namespace: default + autoGenerate: true diff --git a/docs/modules/ROOT/examples/example-cluster-tls-encryption.yaml b/docs/modules/ROOT/examples/example-cluster-tls-encryption.yaml new file mode 100644 index 00000000..1f415f18 --- /dev/null +++ b/docs/modules/ROOT/examples/example-cluster-tls-encryption.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: simple-zk +spec: + version: 3.8.0 + config: + tls: + secretClass: tls # <1> + quorumTlsSecretClass: tls # <2> + servers: + roleGroups: + default: + replicas: 3 diff --git a/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml b/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml new file mode 100644 index 00000000..c0f567ce --- /dev/null +++ b/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: tls +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-ca + namespace: default + autoGenerate: true \ No newline at end of file diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index e4449f48..3f53e1ca 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -6,41 +6,77 @@ If you are not installing the operator using Helm then after installation the CR To create a three-node Apache ZooKeeper (v3.8.0) cluster: - - $ cat zookeeper.yaml - --- - apiVersion: zookeeper.stackable.tech/v1alpha1 - kind: ZookeeperCluster - metadata: - name: simple-zk - spec: - version: 3.8.0 - servers: - roleGroups: - default: - replicas: 3 - config: {} - $ kubectl apply -f zookeeper.yaml +[source,yaml] +---- +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: simple-zk +spec: + version: 3.8.0 + servers: + roleGroups: + default: + replicas: 3 + config: {} +---- Afterwards, a xref:znodes.adoc[ZNode] can be created: - $ cat znode.yaml - --- - apiVersion: zookeeper.stackable.tech/v1alpha1 - kind: ZookeeperZnode - metadata: - name: simple-znode - spec: - clusterRef: - name: simple-zk - namespace: default - $ kubectl apply -f znode.yaml +[source,yaml] +---- +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: simple-znode +spec: + clusterRef: + name: simple-zk + namespace: default +---- Finally, a ConfigMap is created, containing a path that a ZooKeeper client can connect to: $ kubectl get configmap simple-znode-nodeport -o yaml $ $ZOOKEEPER_HOME/bin/zkCli.sh -server $(kubectl get configmap simple-znode-nodeport -o jsonpath='{.data.ZOOKEEPER}') - + +== Encryption + +The quorum and client communication are encrypted by default via TLS. This requires the https://github.com/stackabletech/secret-operator[Secret Operator] to be present in order to provide certificates. The utilized certificates can be changed in a top-level config. + +[source,yaml] +---- +include::example$example-cluster-tls-encryption.yaml[] +---- +<1> The `tls.secretClass` refers to the client-to-server encryption. Defaults to the `tls` secret. +<2> The `quorumTlsSecretClass` refers to the server-to-server quorum encryption. Defaults to the `tls` secret. + +The `tls` secret is deployed from the https://github.com/stackabletech/secret-operator[Secret Operator] and looks like this: + +[source,yaml] +---- +include::example$example-secret-operator-tls-secret.yaml[] +---- + +You can create your own secrets and reference them e.g. in the `tls.secretClass` to use different certificates. + +== Authentication + +The quorum or server-to-server communication is authenticated via TLS per default. In order to enforce TLS authentication for client-to-server communication, you can set an `AuthenticationClass` reference in the custom resource provided by the https://github.com/stackabletech/commons-operator[Commons Operator]. + +[source,yaml] +---- +include::example$example-cluster-tls-authentication.yaml[] +---- +<1> The `config.clientAuthentication.authenticationClass` can be set to use TLS for authentication. This is optional. +<2> The referenced `AuthenticationClass` that references a `SecretClass` to provide certificates. +<3> The reference to a `SecretClass`. +<4> The `SecretClass` that is referenced by the `AuthenticationClass` in order to provide certificates. + +WARNING: Due to a https://issues.apache.org/jira/browse/ZOOKEEPER-4276[bug] in ZooKeeper, the `clientPort` property in combination with `client.portUnification=true` is used instead of the `secureClientPort`. This means that unencrypted and unauthenticated access to the ZooKeeper cluster is still possible. + == Monitoring The managed ZooKeeper instances are automatically configured to export Prometheus metrics. See From 1b556520e16b56023baa8522a1478f1b71cd9fff Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 6 May 2022 15:06:05 +0200 Subject: [PATCH 08/27] writing password via script instead of config map --- rust/crd/src/lib.rs | 24 ++------ rust/operator-binary/Cargo.toml | 3 +- rust/operator-binary/src/auth_config.rs | 67 +++++++++++++++++++---- rust/operator-binary/src/zk_controller.rs | 19 +++++-- 4 files changed, 77 insertions(+), 36 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 7967660d..99f37071 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -14,12 +14,10 @@ pub const SECURE_CLIENT_PORT: u16 = 2282; pub const STACKABLE_DATA_DIR: &str = "/stackable/data"; pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; +pub const STACKABLE_RW_CONFIG_DIR: &str = "/stackable/rwconfig"; pub const QUORUM_TLS_DIR: &str = "/stackable/tls/quorum"; pub const CLIENT_TLS_DIR: &str = "/stackable/tls/client"; -// TODO: Generate? Currently this is written in the ConfigMap. -// It probably makes sense to add that after mounting the CM? -pub const TLS_STORE_SECRET: &str = "My_5t4ck4ble_53cr4T"; const DEFAULT_SECRET_CLASS: &str = "tls"; @@ -222,22 +220,16 @@ impl Configuration for ZookeeperConfig { Self::SSL_AUTH_PROVIDER_X509.to_string(), Some("org.apache.zookeeper.server.auth.X509AuthenticationProvider".to_string()), ); + // The keystore and truststore passwords should not be in the configmap and are generated + // and written later via script in the init container result.insert( Self::SSL_QUORUM_KEY_STORE_LOCATION.to_string(), Some(format!("{dir}/keystore.p12", dir = QUORUM_TLS_DIR)), ); - result.insert( - Self::SSL_QUORUM_KEY_STORE_PASSWORD.to_string(), - Some(TLS_STORE_SECRET.to_string()), - ); result.insert( Self::SSL_QUORUM_TRUST_STORE_LOCATION.to_string(), Some(format!("{dir}/truststore.p12", dir = QUORUM_TLS_DIR)), ); - result.insert( - Self::SSL_QUORUM_TRUST_STORE_PASSWORD.to_string(), - Some(TLS_STORE_SECRET.to_string()), - ); // Client TLS if resource.is_client_secure() { @@ -280,22 +272,16 @@ impl Configuration for ZookeeperConfig { ); // END TICKET + // The keystore and truststore passwords should not be in the configmap and are generated + // and written later via script in the init container result.insert( Self::SSL_KEY_STORE_LOCATION.to_string(), Some(format!("{dir}/keystore.p12", dir = CLIENT_TLS_DIR)), ); - result.insert( - Self::SSL_KEY_STORE_PASSWORD.to_string(), - Some(TLS_STORE_SECRET.to_string()), - ); result.insert( Self::SSL_TRUST_STORE_LOCATION.to_string(), Some(format!("{dir}/truststore.p12", dir = CLIENT_TLS_DIR)), ); - result.insert( - Self::SSL_TRUST_STORE_PASSWORD.to_string(), - Some(TLS_STORE_SECRET.to_string()), - ); // Check if we need to enable authentication if resource.client_tls_authentication_class().is_some() { result.insert(Self::SSL_CLIENT_AUTH.to_string(), Some("need".to_string())); diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 454ae76c..3b2ae6e2 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -13,6 +13,7 @@ clap = { version = "3.1.8", features = ["derive"] } failure = "0.1.8" fnv = "1.0.7" futures = { version = "0.3.21", features = ["compat"] } +pin-project = "1.0.10" semver = "1.0.7" serde = "1.0.136" serde_json = "1.0.79" @@ -24,7 +25,7 @@ tokio01 = { version = "0.1.22", package = "tokio" } tokio-executor = "0.1.10" tokio-zookeeper = "0.1.3" tracing = "0.1.33" -pin-project = "1.0.10" + stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.19.0" } stackable-zookeeper-crd = { path = "../crd" } diff --git a/rust/operator-binary/src/auth_config.rs b/rust/operator-binary/src/auth_config.rs index 4113c998..da1059a0 100644 --- a/rust/operator-binary/src/auth_config.rs +++ b/rust/operator-binary/src/auth_config.rs @@ -1,25 +1,56 @@ use stackable_zookeeper_crd::{ - ZookeeperCluster, CLIENT_TLS_DIR, QUORUM_TLS_DIR, STACKABLE_DATA_DIR, TLS_STORE_SECRET, + ZookeeperCluster, ZookeeperConfig, CLIENT_TLS_DIR, QUORUM_TLS_DIR, STACKABLE_CONFIG_DIR, + STACKABLE_DATA_DIR, STACKABLE_RW_CONFIG_DIR, }; +const STORE_PASSWORD_ENV: &str = "STORE_PASSWORD"; + pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { let mut args = vec![]; - args.extend(create_key_and_trust_store_cmd( - QUORUM_TLS_DIR, - TLS_STORE_SECRET, - )); + // copy config files to a writeable empty folder + // in order to set key and truststore pws later + args.extend(vec![ + format!( + "echo copying {conf} to {rw_conf}", + conf = STACKABLE_CONFIG_DIR, + rw_conf = STACKABLE_RW_CONFIG_DIR + ), + format!( + "cp -RL {conf}/* {rw_conf}", + conf = STACKABLE_CONFIG_DIR, + rw_conf = STACKABLE_RW_CONFIG_DIR + ), + ]); + + args.push(generate_password()); + args.extend(create_key_and_trust_store_cmd(QUORUM_TLS_DIR)); + args.extend(vec![ + write_store_password_to_config(ZookeeperConfig::SSL_QUORUM_KEY_STORE_PASSWORD), + write_store_password_to_config(ZookeeperConfig::SSL_QUORUM_TRUST_STORE_PASSWORD), + ]); if zk.is_client_secure() { - args.extend(create_key_and_trust_store_cmd( - CLIENT_TLS_DIR, - TLS_STORE_SECRET, - )); + args.push(generate_password()); + args.extend(create_key_and_trust_store_cmd(CLIENT_TLS_DIR)); + + args.extend(vec![ + write_store_password_to_config(ZookeeperConfig::SSL_KEY_STORE_PASSWORD), + write_store_password_to_config(ZookeeperConfig::SSL_TRUST_STORE_PASSWORD), + ]); } args.extend([ format!("chown stackable:stackable {dir}", dir = STACKABLE_DATA_DIR), format!("chmod a=,u=rwX {dir}", dir = STACKABLE_DATA_DIR), + format!( + "chown -R stackable:stackable {rwconf_directory}", + rwconf_directory = STACKABLE_RW_CONFIG_DIR + ), + format!( + "chmod -R a=,u=rwX {rwconf_directory}", + rwconf_directory = STACKABLE_RW_CONFIG_DIR + ), format!( "expr $MYID_OFFSET + $(echo $POD_NAME | sed 's/.*-//') > {dir}/myid", dir = STACKABLE_DATA_DIR @@ -29,14 +60,26 @@ pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { args.join(" && ") } -fn create_key_and_trust_store_cmd(directory: &str, password: &str) -> Vec { +fn generate_password() -> String { + format!("export {STORE_PASSWORD_ENV}=$(echo $RANDOM | md5sum | head -c 20)",) +} + +fn write_store_password_to_config(property: &str) -> String { + format!( + "echo {property}=${STORE_PASSWORD_ENV} >> {rwconf}/zoo.cfg", + property = property, + rwconf = STACKABLE_RW_CONFIG_DIR + ) +} + +fn create_key_and_trust_store_cmd(directory: &str) -> Vec { vec![ format!("echo [{dir}] Storing password", dir = directory), - format!("echo {pw} > {dir}/password", pw = password, dir = directory), + format!("echo ${STORE_PASSWORD_ENV} > {dir}/password", dir = directory), format!("echo [{dir}] Cleaning up truststore - just in case", dir = directory), format!("rm -f {dir}/truststore.p12", dir = directory), format!("echo [{dir}] Creating truststore", dir = directory), - format!("keytool -importcert -file {dir}/ca.crt -keystore {dir}/truststore.p12 -storetype pkcs12 -noprompt -alias ca_cert -storepass {pw}", pw = password, dir = directory), + format!("keytool -importcert -file {dir}/ca.crt -keystore {dir}/truststore.p12 -storetype pkcs12 -noprompt -alias ca_cert -storepass ${STORE_PASSWORD_ENV}", dir = directory), format!("echo [{dir}] Creating certificate chain", dir = directory), format!("cat {dir}/ca.crt {dir}/tls.crt > {dir}/chain.crt", dir = directory), format!("echo [{dir}] Creating keystore", dir = directory), diff --git a/rust/operator-binary/src/zk_controller.rs b/rust/operator-binary/src/zk_controller.rs index a43fd833..b7fdd818 100644 --- a/rust/operator-binary/src/zk_controller.rs +++ b/rust/operator-binary/src/zk_controller.rs @@ -16,7 +16,7 @@ use crate::{ use fnv::FnvHasher; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::commons::authentication::AuthenticationClassProvider; -use stackable_operator::k8s_openapi::api::core::v1::CSIVolumeSource; +use stackable_operator::k8s_openapi::api::core::v1::{CSIVolumeSource, EmptyDirVolumeSource}; use stackable_operator::kube::api::DynamicObject; use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, @@ -47,7 +47,7 @@ use stackable_operator::{ }; use stackable_zookeeper_crd::{ ZookeeperCluster, ZookeeperClusterStatus, ZookeeperRole, CLIENT_TLS_DIR, QUORUM_TLS_DIR, - STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, + STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, STACKABLE_RW_CONFIG_DIR, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -485,7 +485,9 @@ fn build_server_rolegroup_statefulset( ..EnvVar::default() }]) .add_volume_mount("data", STACKABLE_DATA_DIR) - .add_volume_mount("quorum-tls", QUORUM_TLS_DIR); + .add_volume_mount("quorum-tls", QUORUM_TLS_DIR) + .add_volume_mount("config", STACKABLE_CONFIG_DIR) + .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR); if zk.is_client_secure() { container_prepare.add_volume_mount("client-tls", CLIENT_TLS_DIR); @@ -503,7 +505,7 @@ fn build_server_rolegroup_statefulset( .args(vec![ "bin/zkServer.sh".to_string(), "start-foreground".to_string(), - format!("{dir}/zoo.cfg", dir = STACKABLE_CONFIG_DIR), + format!("{dir}/zoo.cfg", dir = STACKABLE_RW_CONFIG_DIR), ]) .add_env_vars(env) // Only allow the global load balancing service to send traffic to pods that are members of the quorum @@ -530,6 +532,7 @@ fn build_server_rolegroup_statefulset( .add_container_port("metrics", 9505) .add_volume_mount("data", STACKABLE_DATA_DIR) .add_volume_mount("config", STACKABLE_CONFIG_DIR) + .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR) .add_volume_mount("quorum-tls", QUORUM_TLS_DIR); if zk.is_client_secure() { @@ -559,6 +562,14 @@ fn build_server_rolegroup_statefulset( }), ..Volume::default() }) + .add_volume(Volume { + empty_dir: Some(EmptyDirVolumeSource { + medium: None, + size_limit: None, + }), + name: "rwconfig".to_string(), + ..Volume::default() + }) .add_volume(Volume { name: "quorum-tls".to_string(), csi: Some(CSIVolumeSource { From d41eee0a1751dbf8f329c1f8580bddc861977e16 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 6 May 2022 15:11:38 +0200 Subject: [PATCH 09/27] added comments --- rust/crd/src/lib.rs | 12 ++++++++---- rust/operator-binary/src/auth_config.rs | 6 ++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 99f37071..ad08839e 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -262,16 +262,20 @@ impl Configuration for ZookeeperConfig { Some(resource.client_port().to_string()), ); result.insert( - Self::SSL_HOST_NAME_VERIFICATION.to_string(), + "client.portUnification".to_string(), Some("true".to_string()), ); + // TODO: Remove clientPort and portUnification in favor of secureClientPort once the bug is fixed + // result.insert( + // Self::SECURE_CLIENT_PORT.to_string(), + // Some(resource.client_port().to_string()), + // ); + // END TICKET result.insert( - "client.portUnification".to_string(), + Self::SSL_HOST_NAME_VERIFICATION.to_string(), Some("true".to_string()), ); - // END TICKET - // The keystore and truststore passwords should not be in the configmap and are generated // and written later via script in the init container result.insert( diff --git a/rust/operator-binary/src/auth_config.rs b/rust/operator-binary/src/auth_config.rs index da1059a0..bf119df4 100644 --- a/rust/operator-binary/src/auth_config.rs +++ b/rust/operator-binary/src/auth_config.rs @@ -8,8 +8,8 @@ const STORE_PASSWORD_ENV: &str = "STORE_PASSWORD"; pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { let mut args = vec![]; - // copy config files to a writeable empty folder - // in order to set key and truststore pws later + // copy config files to a writeable empty folder in order to set key and + // truststore passwords in the initcontainer via script args.extend(vec![ format!( "echo copying {conf} to {rw_conf}", @@ -23,6 +23,7 @@ pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { ), ]); + // Quorum args.push(generate_password()); args.extend(create_key_and_trust_store_cmd(QUORUM_TLS_DIR)); args.extend(vec![ @@ -30,6 +31,7 @@ pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { write_store_password_to_config(ZookeeperConfig::SSL_QUORUM_TRUST_STORE_PASSWORD), ]); + // Client if zk.is_client_secure() { args.push(generate_password()); args.extend(create_key_and_trust_store_cmd(CLIENT_TLS_DIR)); From 5073e7c926fb5ead9588bcd24a773b7ebb166510 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 6 May 2022 15:16:29 +0200 Subject: [PATCH 10/27] changed password generation --- rust/operator-binary/src/auth_config.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/auth_config.rs b/rust/operator-binary/src/auth_config.rs index bf119df4..f774b354 100644 --- a/rust/operator-binary/src/auth_config.rs +++ b/rust/operator-binary/src/auth_config.rs @@ -63,7 +63,8 @@ pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { } fn generate_password() -> String { - format!("export {STORE_PASSWORD_ENV}=$(echo $RANDOM | md5sum | head -c 20)",) + // taken from https://unix.stackexchange.com/questions/230673/how-to-generate-a-random-string + format!("export {STORE_PASSWORD_ENV}=$(tr -dc A-Za-z0-9 String { From 331b006b33894d86ee0a01f50a8867bcbaacbdfb Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 6 May 2022 16:13:15 +0200 Subject: [PATCH 11/27] adapted changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29bdd959..476d8eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,18 +11,18 @@ All notable changes to this project will be documented in this file. a single namespace to watch ([#434]). - Support for ZooKeeper 3.8.0 added ([#464]). - Integration tests for all supported ZooKeeper versions added ([#464]). -- TLS encryption and authentication support for quorum and client ([#xxx]). +- TLS encryption and authentication support for quorum and client ([#479]). ### Changed -- Operator-rs: 0.10.0 -> 0.19.0 ([#408], [#431], [#434], [#454], [#xxx]). +- Operator-rs: 0.10.0 -> 0.19.0 ([#408], [#431], [#434], [#454], [#479]). [#408]: https://github.com/stackabletech/zookeeper-operator/pull/408 [#431]: https://github.com/stackabletech/zookeeper-operator/pull/431 [#434]: https://github.com/stackabletech/zookeeper-operator/pull/434 [#454]: https://github.com/stackabletech/zookeeper-operator/pull/454 [#464]: https://github.com/stackabletech/zookeeper-operator/pull/464 -[#xxx]: https://github.com/stackabletech/zookeeper-operator/pull/xxx +[#479]: https://github.com/stackabletech/zookeeper-operator/pull/479 ## [0.9.0] - 2022-02-14 From 2d488d3cb5639b8ec443c8eea934e060842c9252 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 6 May 2022 16:13:34 +0200 Subject: [PATCH 12/27] renamed auth_config to command --- rust/operator-binary/src/{auth_config.rs => command.rs} | 0 rust/operator-binary/src/main.rs | 2 +- rust/operator-binary/src/zk_controller.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename rust/operator-binary/src/{auth_config.rs => command.rs} (100%) diff --git a/rust/operator-binary/src/auth_config.rs b/rust/operator-binary/src/command.rs similarity index 100% rename from rust/operator-binary/src/auth_config.rs rename to rust/operator-binary/src/command.rs diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index b0fdf881..cc88a41b 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,4 +1,4 @@ -mod auth_config; +mod command; mod discovery; mod utils; mod zk_controller; diff --git a/rust/operator-binary/src/zk_controller.rs b/rust/operator-binary/src/zk_controller.rs index b7fdd818..99294527 100644 --- a/rust/operator-binary/src/zk_controller.rs +++ b/rust/operator-binary/src/zk_controller.rs @@ -9,7 +9,7 @@ use std::{ }; use crate::{ - auth_config::create_init_container_command_args, + command::create_init_container_command_args, discovery::{self, build_discovery_configmaps}, ObjectRef, APP_NAME, }; From 26b655eba0ecfca26d43eb2f13adef1d75ecbddc Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 6 May 2022 16:25:14 +0200 Subject: [PATCH 13/27] attempt to fix yaml lints --- deploy/config-spec/properties.yaml | 6 ++--- ...mple-cluster-tls-authentication-class.yaml | 9 ++++++++ ...ple-cluster-tls-authentication-secret.yaml | 13 +++++++++++ .../example-cluster-tls-authentication.yaml | 23 ------------------- .../example-secret-operator-tls-secret.yaml | 2 +- docs/modules/ROOT/pages/usage.adoc | 10 ++++---- 6 files changed, 32 insertions(+), 31 deletions(-) create mode 100644 docs/modules/ROOT/examples/example-cluster-tls-authentication-class.yaml create mode 100644 docs/modules/ROOT/examples/example-cluster-tls-authentication-secret.yaml diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index c78894d8..dff01dc0 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -369,9 +369,9 @@ properties: required: false asOfVersion: "3.5.5" description: "The password to unlock the client truststore." - ########################## - # Common TLS - ########################## + ########################## + # Common TLS + ########################## - property: &authProviderX509 propertyNames: - name: "authProvider.x509" diff --git a/docs/modules/ROOT/examples/example-cluster-tls-authentication-class.yaml b/docs/modules/ROOT/examples/example-cluster-tls-authentication-class.yaml new file mode 100644 index 00000000..1240b539 --- /dev/null +++ b/docs/modules/ROOT/examples/example-cluster-tls-authentication-class.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: zk-client-tls # <2> +spec: + provider: + tls: + clientCertSecretClass: zk-client-auth-secret # <3> diff --git a/docs/modules/ROOT/examples/example-cluster-tls-authentication-secret.yaml b/docs/modules/ROOT/examples/example-cluster-tls-authentication-secret.yaml new file mode 100644 index 00000000..e7bcd065 --- /dev/null +++ b/docs/modules/ROOT/examples/example-cluster-tls-authentication-secret.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: zk-client-auth-secret # <4> +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-zk-client-ca + namespace: default + autoGenerate: true diff --git a/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml b/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml index cc0a6a2e..45597957 100644 --- a/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml +++ b/docs/modules/ROOT/examples/example-cluster-tls-authentication.yaml @@ -1,4 +1,3 @@ - --- apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperCluster @@ -16,25 +15,3 @@ spec: roleGroups: default: replicas: 3 ---- -apiVersion: authentication.stackable.tech/v1alpha1 -kind: AuthenticationClass -metadata: - name: zk-client-tls # <2> -spec: - provider: - tls: - clientCertSecretClass: zk-client-auth-secret # <3> ---- -apiVersion: secrets.stackable.tech/v1alpha1 -kind: SecretClass -metadata: - name: zk-client-auth-secret # <4> -spec: - backend: - autoTls: - ca: - secret: - name: secret-provisioner-tls-zk-client-ca - namespace: default - autoGenerate: true diff --git a/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml b/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml index c0f567ce..e6f27e9b 100644 --- a/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml +++ b/docs/modules/ROOT/examples/example-secret-operator-tls-secret.yaml @@ -10,4 +10,4 @@ spec: secret: name: secret-provisioner-tls-ca namespace: default - autoGenerate: true \ No newline at end of file + autoGenerate: true diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 96e52c8f..c1abaf35 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -47,18 +47,18 @@ Finally, a ConfigMap is created, containing a path that a ZooKeeper client can c The quorum and client communication are encrypted by default via TLS. This requires the https://github.com/stackabletech/secret-operator[Secret Operator] to be present in order to provide certificates. The utilized certificates can be changed in a top-level config. [source,yaml] ----- +==== include::example$example-cluster-tls-encryption.yaml[] ----- +==== <1> The `tls.secretClass` refers to the client-to-server encryption. Defaults to the `tls` secret. <2> The `quorumTlsSecretClass` refers to the server-to-server quorum encryption. Defaults to the `tls` secret. The `tls` secret is deployed from the https://github.com/stackabletech/secret-operator[Secret Operator] and looks like this: [source,yaml] ----- +==== include::example$example-secret-operator-tls-secret.yaml[] ----- +==== You can create your own secrets and reference them e.g. in the `tls.secretClass` to use different certificates. @@ -69,6 +69,8 @@ The quorum or server-to-server communication is authenticated via TLS per defaul [source,yaml] ---- include::example$example-cluster-tls-authentication.yaml[] +include::example$example-cluster-tls-authentication-class.yaml[] +include::example$example-cluster-tls-authentication-secret.yaml[] ---- <1> The `config.clientAuthentication.authenticationClass` can be set to use TLS for authentication. This is optional. <2> The referenced `AuthenticationClass` that references a `SecretClass` to provide certificates. From 47699b6f1a24b6ccfa6fa442f9398819e317baec Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 6 May 2022 16:55:44 +0200 Subject: [PATCH 14/27] regenerated charts --- .../configs/properties.yaml | 312 +++++++++++++++-- deploy/helm/zookeeper-operator/crds/crds.yaml | 3 + deploy/manifests/configmap.yaml | 314 ++++++++++++++++-- deploy/manifests/crds.yaml | 3 + 4 files changed, 575 insertions(+), 57 deletions(-) diff --git a/deploy/helm/zookeeper-operator/configs/properties.yaml b/deploy/helm/zookeeper-operator/configs/properties.yaml index aebefcde..dff01dc0 100644 --- a/deploy/helm/zookeeper-operator/configs/properties.yaml +++ b/deploy/helm/zookeeper-operator/configs/properties.yaml @@ -9,6 +9,7 @@ spec: - unit: &unitPort name: "port" regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" + properties: - property: &tickTime propertyNames: @@ -29,28 +30,6 @@ properties: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "The basic time unit in milliseconds used by ZooKeeper. It is used to do heartbeats and the minimum session timeout will be twice the tickTime." - - property: &clientPort - propertyNames: - - name: "clientPort" - kind: - type: "file" - file: "zoo.cfg" - datatype: - type: "integer" - unit: *unitPort - min: "1024" - max: "65535" - defaultValues: - - value: "2181" - recommendedValues: - - value: "2181" - roles: - - name: "server" - required: true - asOfVersion: "0.0.0" - comment: "See QuorumPeerConfig.java, I'm unsure what happens when this is set to 0, it might work, it might not" - description: "The port to listen for client connections; that is, the port that clients attempt to connect to." - - property: &dataDir propertyNames: - name: "dataDir" @@ -109,11 +88,34 @@ properties: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "Amount of time, in ticks (see `tickTime`), to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they will be dropped. In other words: The number of ticks that can pass between sending a request and getting an acknowledgment before a follower is dropped." - - property: &metricsPort + ########################## + # Ports + ########################## + - property: &clientPort + propertyNames: + - name: "clientPort" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "integer" + unit: *unitPort + min: "1024" + max: "65535" + defaultValues: + - value: "2181" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "The plaintext port to listen on for client connections." + + - property: &secureClientPort propertyNames: - - name: "metricsPort" + - name: "secureClientPort" kind: - type: "env" + type: "file" + file: "zoo.cfg" datatype: type: "integer" unit: *unitPort @@ -123,9 +125,9 @@ properties: - name: "server" required: false asOfVersion: "0.0.0" - description: "The port where ZooKeeper metrics are exposed as a Prometheus endpoint." + description: "The secure port to listen on for secure client connections using TLS/SSL." - - property: &admin_serverPort + - property: &adminServerPort propertyNames: - name: "admin.serverPort" kind: @@ -142,4 +144,258 @@ properties: - name: "server" required: true asOfVersion: "0.0.0" - description: "The zookeeper admin server port." \ No newline at end of file + description: "The port the embedded Jetty server listens on." + + ########################## + # Quorum TLS + ########################## + - property: &sslQuorum + propertyNames: + - name: "sslQuorum" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + defaultValues: + - value: "false" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Enables encrypted quorum communication." + + - property: &sslQuorumClientAuth + propertyNames: + - name: "ssl.quorum.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from clients." + + - property: &sslQuorumHostNameVerification + propertyNames: + - name: "ssl.quorum.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in quorum TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslQuorumKeyStoreLocation + propertyNames: + - name: "ssl.quorum.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumKeyStorePassword + propertyNames: + - name: "ssl.quorum.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum keystore." + + - property: &sslQuorumTrustStoreLocation + propertyNames: + - name: "ssl.quorum.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumTrustStorePassword + propertyNames: + - name: "ssl.quorum.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum truststore." + + ########################## + # Client TLS + ########################## + - property: &sslClientAuth + propertyNames: + - name: "ssl.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from clients." + + - property: &sslHostNameVerification + propertyNames: + - name: "ssl.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in client TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslKeyStoreLocation + propertyNames: + - name: "ssl.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for client TLS connections." + + - property: &sslKeyStorePassword + propertyNames: + - name: "ssl.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client keystore." + + - property: &sslTrustStoreLocation + propertyNames: + - name: "ssl.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the remote credentials to be used for client TLS connections." + + - property: &sslTrustStorePassword + propertyNames: + - name: "ssl.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client truststore." + ########################## + # Common TLS + ########################## + - property: &authProviderX509 + propertyNames: + - name: "authProvider.x509" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "Used for secure client authentication." + + - property: &serverCnxnFactory + propertyNames: + - name: "serverCnxnFactory" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "This should be set to NettyServerCnxnFactory in order to use TLS based server communication." diff --git a/deploy/helm/zookeeper-operator/crds/crds.yaml b/deploy/helm/zookeeper-operator/crds/crds.yaml index ec9c30ac..3aa10904 100644 --- a/deploy/helm/zookeeper-operator/crds/crds.yaml +++ b/deploy/helm/zookeeper-operator/crds/crds.yaml @@ -29,6 +29,7 @@ spec: nullable: true properties: clientAuthentication: + description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" nullable: true properties: authenticationClass: @@ -37,8 +38,10 @@ spec: - authenticationClass type: object quorumTlsSecretClass: + description: "Only affects quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" type: string tls: + description: "Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: \"tls\".to_string() }." nullable: true properties: secretClass: diff --git a/deploy/manifests/configmap.yaml b/deploy/manifests/configmap.yaml index 281ebf16..b4ec3711 100644 --- a/deploy/manifests/configmap.yaml +++ b/deploy/manifests/configmap.yaml @@ -1,7 +1,7 @@ --- apiVersion: v1 data: - properties.yaml: |- + properties.yaml: | version: 0.1.0 spec: units: @@ -13,6 +13,7 @@ data: - unit: &unitPort name: "port" regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" + properties: - property: &tickTime propertyNames: @@ -33,28 +34,6 @@ data: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "The basic time unit in milliseconds used by ZooKeeper. It is used to do heartbeats and the minimum session timeout will be twice the tickTime." - - property: &clientPort - propertyNames: - - name: "clientPort" - kind: - type: "file" - file: "zoo.cfg" - datatype: - type: "integer" - unit: *unitPort - min: "1024" - max: "65535" - defaultValues: - - value: "2181" - recommendedValues: - - value: "2181" - roles: - - name: "server" - required: true - asOfVersion: "0.0.0" - comment: "See QuorumPeerConfig.java, I'm unsure what happens when this is set to 0, it might work, it might not" - description: "The port to listen for client connections; that is, the port that clients attempt to connect to." - - property: &dataDir propertyNames: - name: "dataDir" @@ -113,11 +92,34 @@ data: comment: "ZK only checks whether the value is 0, all other values (including negative ones) are considered valid, we disallow negative values here, see QuorumPeerConfig.java" description: "Amount of time, in ticks (see `tickTime`), to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they will be dropped. In other words: The number of ticks that can pass between sending a request and getting an acknowledgment before a follower is dropped." - - property: &metricsPort + ########################## + # Ports + ########################## + - property: &clientPort + propertyNames: + - name: "clientPort" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "integer" + unit: *unitPort + min: "1024" + max: "65535" + defaultValues: + - value: "2181" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "The plaintext port to listen on for client connections." + + - property: &secureClientPort propertyNames: - - name: "metricsPort" + - name: "secureClientPort" kind: - type: "env" + type: "file" + file: "zoo.cfg" datatype: type: "integer" unit: *unitPort @@ -127,9 +129,9 @@ data: - name: "server" required: false asOfVersion: "0.0.0" - description: "The port where ZooKeeper metrics are exposed as a Prometheus endpoint." + description: "The secure port to listen on for secure client connections using TLS/SSL." - - property: &admin_serverPort + - property: &adminServerPort propertyNames: - name: "admin.serverPort" kind: @@ -146,7 +148,261 @@ data: - name: "server" required: true asOfVersion: "0.0.0" - description: "The zookeeper admin server port." + description: "The port the embedded Jetty server listens on." + + ########################## + # Quorum TLS + ########################## + - property: &sslQuorum + propertyNames: + - name: "sslQuorum" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + defaultValues: + - value: "false" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Enables encrypted quorum communication." + + - property: &sslQuorumClientAuth + propertyNames: + - name: "ssl.quorum.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from clients." + + - property: &sslQuorumHostNameVerification + propertyNames: + - name: "ssl.quorum.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in quorum TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslQuorumKeyStoreLocation + propertyNames: + - name: "ssl.quorum.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumKeyStorePassword + propertyNames: + - name: "ssl.quorum.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum keystore." + + - property: &sslQuorumTrustStoreLocation + propertyNames: + - name: "ssl.quorum.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/quorum" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the local credentials to be used for quorum TLS connections." + + - property: &sslQuorumTrustStorePassword + propertyNames: + - name: "ssl.quorum.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the quorum truststore." + + ########################## + # Client TLS + ########################## + - property: &sslClientAuth + propertyNames: + - name: "ssl.clientAuth" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + allowedValues: + - "none" + - "want" + - "need" + defaultValues: + - value: "need" + recommendedValues: + - value: "need" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies options to authenticate ssl connections from clients." + + - property: &sslHostNameVerification + propertyNames: + - name: "ssl.hostnameVerification" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "bool" + recommendedValues: + - value: "true" + roles: + - name: "server" + required: false + asOfVersion: "3.5.7" + description: "Specifies whether the hostname verification is enabled in client TLS negotiation process. Disabling it only recommended for testing purposes." + + - property: &sslKeyStoreLocation + propertyNames: + - name: "ssl.keyStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java keystore containing the local credentials to be used for client TLS connections." + + - property: &sslKeyStorePassword + propertyNames: + - name: "ssl.keyStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client keystore." + + - property: &sslTrustStoreLocation + propertyNames: + - name: "ssl.trustStore.location" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + unit: *unitDirectory + defaultValues: + - value: "/stackable/tls/client" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "Specifies the file path to a Java truststore containing the remote credentials to be used for client TLS connections." + + - property: &sslTrustStorePassword + propertyNames: + - name: "ssl.trustStore.password" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "3.5.5" + description: "The password to unlock the client truststore." + ########################## + # Common TLS + ########################## + - property: &authProviderX509 + propertyNames: + - name: "authProvider.x509" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "Used for secure client authentication." + + - property: &serverCnxnFactory + propertyNames: + - name: "serverCnxnFactory" + kind: + type: "file" + file: "zoo.cfg" + datatype: + type: "string" + roles: + - name: "server" + required: false + asOfVersion: "0.0.0" + description: "This should be set to NettyServerCnxnFactory in order to use TLS based server communication." kind: ConfigMap metadata: name: zookeeper-operator-configmap diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 483f10d2..b671142a 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -30,6 +30,7 @@ spec: nullable: true properties: clientAuthentication: + description: "Only affects client connections. This setting controls: - If clients need to authenticate themselves against the server via TLS - Which ca.crt to use when validating the provided client certs Defaults to `None`" nullable: true properties: authenticationClass: @@ -38,8 +39,10 @@ spec: - authenticationClass type: object quorumTlsSecretClass: + description: "Only affects quorum communication. Use mutual verification between Zookeeper Nodes (mandatory). This setting controls: - Which cert the servers should use to authenticate themselves against other servers - Which ca.crt to use when validating the other server" type: string tls: + description: "Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client Defaults to `TlsSecretClass` { secret_class: \"tls\".to_string() }." nullable: true properties: secretClass: From b533e7a206abac3edc0ca094f0749e4e9a5d0dd1 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 9 May 2022 13:30:09 +0200 Subject: [PATCH 15/27] added test for tls --- .../kuttl/smoke/00-install-zookeeper.yaml.j2 | 28 ++++++++++ tests/templates/kuttl/smoke/02-assert.yaml | 1 + .../smoke/02-prepare-test-zookeeper.yaml | 1 + tests/templates/kuttl/smoke/test_tls.sh | 54 +++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100755 tests/templates/kuttl/smoke/test_tls.sh diff --git a/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 b/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 index 633d42b2..36718c17 100644 --- a/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/smoke/00-install-zookeeper.yaml.j2 @@ -4,6 +4,12 @@ kind: ZookeeperCluster metadata: name: test-zk spec: + config: + tls: + secretClass: tls + clientAuthentication: + authenticationClass: zk-client-tls + quorumTlsSecretClass: tls servers: roleGroups: primary: @@ -17,6 +23,28 @@ spec: version: {{ test_scenario['values']['zookeeper'] }} stopped: false --- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: zk-client-tls +spec: + provider: + tls: + clientCertSecretClass: zk-client-auth-secret +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: zk-client-auth-secret +spec: + backend: + autoTls: + ca: + secret: + name: secret-provisioner-tls-zk-client-ca + namespace: default + autoGenerate: true +--- apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperZnode metadata: diff --git a/tests/templates/kuttl/smoke/02-assert.yaml b/tests/templates/kuttl/smoke/02-assert.yaml index c184732b..140232f1 100644 --- a/tests/templates/kuttl/smoke/02-assert.yaml +++ b/tests/templates/kuttl/smoke/02-assert.yaml @@ -5,3 +5,4 @@ metadata: name: test-regorule commands: - script: kubectl exec -n $NAMESPACE zk-test-helper-0 -- python /tmp/test_zookeeper.py -n $NAMESPACE + - script: kubectl exec -n $NAMESPACE test-zk-server-primary-0 -- /tmp/test_tls.sh $NAMESPACE diff --git a/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml b/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml index e0d3f04a..1e42f3c5 100644 --- a/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml +++ b/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml @@ -5,3 +5,4 @@ commands: - script: kubectl cp -n $NAMESPACE ./test_zookeeper.py zk-test-helper-0:/tmp - script: kubectl cp -n $NAMESPACE ./requirements.txt zk-test-helper-0:/tmp - script: kubectl exec -n $NAMESPACE zk-test-helper-0 -- pip install --user -r /tmp/requirements.txt + - script: kubectl cp -n $NAMESPACE ./test_tls.sh test-zk-server-primary-0:/tmp \ No newline at end of file diff --git a/tests/templates/kuttl/smoke/test_tls.sh b/tests/templates/kuttl/smoke/test_tls.sh new file mode 100755 index 00000000..a0f8da4c --- /dev/null +++ b/tests/templates/kuttl/smoke/test_tls.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +NAMESPACE=$1 +# test the plaintext unsecured connection +COMMAND="/stackable/zookeeper/bin/zkCli.sh -server test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2282 ls / >> file.log" +$COMMAND + +if [[ $? != 0 ]]; +then + echo "Could not establish unsecure connection..." + exit 1 +fi +echo "Unsecure client connection established successfully!" + +# we set the correct client tls credentials and expect to be able to connect +export CLIENT_STORE_SECRET=$(cat /stackable/rwconfig/zoo.cfg | grep "ssl.keyStore.password" | cut -d "=" -f2) +export CLIENT_JVMFLAGS=" +-Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider +-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty +-Dzookeeper.client.secure=true +-Dzookeeper.ssl.keyStore.location=/stackable/tls/client/keystore.p12 +-Dzookeeper.ssl.keyStore.password=${CLIENT_STORE_SECRET} +-Dzookeeper.ssl.trustStore.location=/stackable/tls/client/truststore.p12 +-Dzookeeper.ssl.trustStore.password=${CLIENT_STORE_SECRET}" + +$COMMAND + +if [[ $? != 0 ]]; +then + echo "Could not establish secure connection..." + exit 1 +fi +echo "Secure and authenticated client connection established successfully!" + +# We set the (wrong) quorum tls credentials and expect to fail (wrong certificate) +export QUORUM_STORE_SECRET=$(cat /stackable/rwconfig/zoo.cfg | grep "ssl.quorum.keyStore.password" | cut -d "=" -f2) +export CLIENT_JVMFLAGS=" +-Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider +-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty +-Dzookeeper.client.secure=true +-Dzookeeper.ssl.keyStore.location=/stackable/tls/quorum/keystore.p12 +-Dzookeeper.ssl.keyStore.password=${QUORUM_STORE_SECRET} +-Dzookeeper.ssl.trustStore.location=/stackable/tls/quorum/truststore.p12 +-Dzookeeper.ssl.trustStore.password=${QUORUM_STORE_SECRET}" + +$COMMAND + +if [[ $? == 0 ]]; +then + echo "Could establish secure connection with wrong certificates... this should not work!" + exit 1 +fi + +echo "TLS test successful!" From e6696cacf9767d959635746f602a47c8b6d9fc6e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 9 May 2022 15:24:12 +0200 Subject: [PATCH 16/27] improved test output --- tests/templates/kuttl/smoke/test_tls.sh | 34 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/templates/kuttl/smoke/test_tls.sh b/tests/templates/kuttl/smoke/test_tls.sh index a0f8da4c..50b4b97f 100755 --- a/tests/templates/kuttl/smoke/test_tls.sh +++ b/tests/templates/kuttl/smoke/test_tls.sh @@ -1,18 +1,25 @@ -#!/bin/bash +#!/usr/bin/env bash +# Usage: test_tls.sh namespace NAMESPACE=$1 +SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2282" + +echo "Start TLS testing..." +############################################################################ # test the plaintext unsecured connection -COMMAND="/stackable/zookeeper/bin/zkCli.sh -server test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2282 ls / >> file.log" -$COMMAND +############################################################################ +/stackable/zookeeper/bin/zkCli.sh -server ${SERVER} ls / &> /dev/null if [[ $? != 0 ]]; then - echo "Could not establish unsecure connection..." + echo "[ERROR] Could not establish unsecure connection!" exit 1 fi -echo "Unsecure client connection established successfully!" +echo "[SUCCESS] Unsecure client connection established!" +############################################################################ # we set the correct client tls credentials and expect to be able to connect +############################################################################ export CLIENT_STORE_SECRET=$(cat /stackable/rwconfig/zoo.cfg | grep "ssl.keyStore.password" | cut -d "=" -f2) export CLIENT_JVMFLAGS=" -Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider @@ -22,17 +29,18 @@ export CLIENT_JVMFLAGS=" -Dzookeeper.ssl.keyStore.password=${CLIENT_STORE_SECRET} -Dzookeeper.ssl.trustStore.location=/stackable/tls/client/truststore.p12 -Dzookeeper.ssl.trustStore.password=${CLIENT_STORE_SECRET}" - -$COMMAND +/stackable/zookeeper/bin/zkCli.sh -server ${SERVER} ls / &> /dev/null if [[ $? != 0 ]]; then - echo "Could not establish secure connection..." + echo "[ERROR] Could not establish secure connection using client certificates!" exit 1 fi -echo "Secure and authenticated client connection established successfully!" +echo "[SUCCESS] Secure and authenticated client connection established!" +############################################################################ # We set the (wrong) quorum tls credentials and expect to fail (wrong certificate) +############################################################################ export QUORUM_STORE_SECRET=$(cat /stackable/rwconfig/zoo.cfg | grep "ssl.quorum.keyStore.password" | cut -d "=" -f2) export CLIENT_JVMFLAGS=" -Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider @@ -43,12 +51,14 @@ export CLIENT_JVMFLAGS=" -Dzookeeper.ssl.trustStore.location=/stackable/tls/quorum/truststore.p12 -Dzookeeper.ssl.trustStore.password=${QUORUM_STORE_SECRET}" -$COMMAND +/stackable/zookeeper/bin/zkCli.sh -server ${SERVER} ls / &> /dev/null if [[ $? == 0 ]]; then - echo "Could establish secure connection with wrong certificates... this should not work!" + echo "[ERROR] Could establish secure connection with quorum certificates (should not be happening)!" exit 1 fi +echo "[SUCCESS] Could not establish secure connection with (wrong) quorum certificates!" -echo "TLS test successful!" +echo "All TLS tests successful!" +exit 0 \ No newline at end of file From f1fc902c41dc74630c01bcb7044ba8dac3cfc92e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 9 May 2022 15:27:35 +0200 Subject: [PATCH 17/27] Update tests/templates/kuttl/smoke/test_tls.sh Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tests/templates/kuttl/smoke/test_tls.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/smoke/test_tls.sh b/tests/templates/kuttl/smoke/test_tls.sh index 50b4b97f..da2e58b0 100755 --- a/tests/templates/kuttl/smoke/test_tls.sh +++ b/tests/templates/kuttl/smoke/test_tls.sh @@ -8,7 +8,7 @@ echo "Start TLS testing..." ############################################################################ # test the plaintext unsecured connection ############################################################################ -/stackable/zookeeper/bin/zkCli.sh -server ${SERVER} ls / &> /dev/null +/stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null if [[ $? != 0 ]]; then From 27e39b80afe3467a7f243a57cc774df1b52e046a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 9 May 2022 15:27:40 +0200 Subject: [PATCH 18/27] Update tests/templates/kuttl/smoke/test_tls.sh Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tests/templates/kuttl/smoke/test_tls.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/smoke/test_tls.sh b/tests/templates/kuttl/smoke/test_tls.sh index da2e58b0..4cbc8c2c 100755 --- a/tests/templates/kuttl/smoke/test_tls.sh +++ b/tests/templates/kuttl/smoke/test_tls.sh @@ -29,7 +29,7 @@ export CLIENT_JVMFLAGS=" -Dzookeeper.ssl.keyStore.password=${CLIENT_STORE_SECRET} -Dzookeeper.ssl.trustStore.location=/stackable/tls/client/truststore.p12 -Dzookeeper.ssl.trustStore.password=${CLIENT_STORE_SECRET}" -/stackable/zookeeper/bin/zkCli.sh -server ${SERVER} ls / &> /dev/null +/stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null if [[ $? != 0 ]]; then From da0698f2855df6b5f9393639af88b7943d3da373 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 9 May 2022 15:27:44 +0200 Subject: [PATCH 19/27] Update tests/templates/kuttl/smoke/test_tls.sh Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tests/templates/kuttl/smoke/test_tls.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/smoke/test_tls.sh b/tests/templates/kuttl/smoke/test_tls.sh index 4cbc8c2c..4b43975d 100755 --- a/tests/templates/kuttl/smoke/test_tls.sh +++ b/tests/templates/kuttl/smoke/test_tls.sh @@ -51,7 +51,7 @@ export CLIENT_JVMFLAGS=" -Dzookeeper.ssl.trustStore.location=/stackable/tls/quorum/truststore.p12 -Dzookeeper.ssl.trustStore.password=${QUORUM_STORE_SECRET}" -/stackable/zookeeper/bin/zkCli.sh -server ${SERVER} ls / &> /dev/null +/stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null if [[ $? == 0 ]]; then From 287e5bc47379b6d9196736c6eb7f0171597b22e7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 9 May 2022 15:42:04 +0200 Subject: [PATCH 20/27] attempt to fix linters #1 --- .../smoke/02-prepare-test-zookeeper.yaml | 2 +- tests/templates/kuttl/smoke/test_tls.sh | 20 +++++++++---------- tests/test-definition.yaml | 6 +++--- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml b/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml index 1e42f3c5..e41ef8fb 100644 --- a/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml +++ b/tests/templates/kuttl/smoke/02-prepare-test-zookeeper.yaml @@ -5,4 +5,4 @@ commands: - script: kubectl cp -n $NAMESPACE ./test_zookeeper.py zk-test-helper-0:/tmp - script: kubectl cp -n $NAMESPACE ./requirements.txt zk-test-helper-0:/tmp - script: kubectl exec -n $NAMESPACE zk-test-helper-0 -- pip install --user -r /tmp/requirements.txt - - script: kubectl cp -n $NAMESPACE ./test_tls.sh test-zk-server-primary-0:/tmp \ No newline at end of file + - script: kubectl cp -n $NAMESPACE ./test_tls.sh test-zk-server-primary-0:/tmp diff --git a/tests/templates/kuttl/smoke/test_tls.sh b/tests/templates/kuttl/smoke/test_tls.sh index 4b43975d..651dd5ec 100755 --- a/tests/templates/kuttl/smoke/test_tls.sh +++ b/tests/templates/kuttl/smoke/test_tls.sh @@ -8,9 +8,7 @@ echo "Start TLS testing..." ############################################################################ # test the plaintext unsecured connection ############################################################################ -/stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null - -if [[ $? != 0 ]]; +if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; then echo "[ERROR] Could not establish unsecure connection!" exit 1 @@ -20,7 +18,8 @@ echo "[SUCCESS] Unsecure client connection established!" ############################################################################ # we set the correct client tls credentials and expect to be able to connect ############################################################################ -export CLIENT_STORE_SECRET=$(cat /stackable/rwconfig/zoo.cfg | grep "ssl.keyStore.password" | cut -d "=" -f2) +CLIENT_STORE_SECRET="$(< /stackable/rwconfig/zoo.cfg grep "ssl.keyStore.password" | cut -d "=" -f2)" +export CLIENT_STORE_SECRET export CLIENT_JVMFLAGS=" -Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty @@ -29,9 +28,9 @@ export CLIENT_JVMFLAGS=" -Dzookeeper.ssl.keyStore.password=${CLIENT_STORE_SECRET} -Dzookeeper.ssl.trustStore.location=/stackable/tls/client/truststore.p12 -Dzookeeper.ssl.trustStore.password=${CLIENT_STORE_SECRET}" -/stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null -if [[ $? != 0 ]]; + +if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; then echo "[ERROR] Could not establish secure connection using client certificates!" exit 1 @@ -41,7 +40,8 @@ echo "[SUCCESS] Secure and authenticated client connection established!" ############################################################################ # We set the (wrong) quorum tls credentials and expect to fail (wrong certificate) ############################################################################ -export QUORUM_STORE_SECRET=$(cat /stackable/rwconfig/zoo.cfg | grep "ssl.quorum.keyStore.password" | cut -d "=" -f2) +QUORUM_STORE_SECRET="$(< /stackable/rwconfig/zoo.cfg grep "ssl.quorum.keyStore.password" | cut -d "=" -f2)" +export QUORUM_STORE_SECRET export CLIENT_JVMFLAGS=" -Dzookeeper.authProvider.x509=org.apache.zookeeper.server.auth.X509AuthenticationProvider -Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty @@ -51,9 +51,7 @@ export CLIENT_JVMFLAGS=" -Dzookeeper.ssl.trustStore.location=/stackable/tls/quorum/truststore.p12 -Dzookeeper.ssl.trustStore.password=${QUORUM_STORE_SECRET}" -/stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null - -if [[ $? == 0 ]]; +if /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; then echo "[ERROR] Could establish secure connection with quorum certificates (should not be happening)!" exit 1 @@ -61,4 +59,4 @@ fi echo "[SUCCESS] Could not establish secure connection with (wrong) quorum certificates!" echo "All TLS tests successful!" -exit 0 \ No newline at end of file +exit 0 diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index b50885b1..8c86d215 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -2,9 +2,9 @@ dimensions: - name: zookeeper values: - - 3.5.8 - - 3.6.3 - - 3.7.0 +# - 3.5.8 +# - 3.6.3 +# - 3.7.0 - 3.8.0 tests: - name: smoke From e8056326d8d13c691095835c8b36264502601105 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 9 May 2022 15:42:23 +0200 Subject: [PATCH 21/27] attempt to fix linters #1 --- tests/test-definition.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 8c86d215..b50885b1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -2,9 +2,9 @@ dimensions: - name: zookeeper values: -# - 3.5.8 -# - 3.6.3 -# - 3.7.0 + - 3.5.8 + - 3.6.3 + - 3.7.0 - 3.8.0 tests: - name: smoke From 2d5c9fb983cf0633e68f913344e2d0575d68612b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 11 May 2022 11:11:57 +0200 Subject: [PATCH 22/27] fix role permissions for authenticationclass --- deploy/helm/zookeeper-operator/templates/roles.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/helm/zookeeper-operator/templates/roles.yaml b/deploy/helm/zookeeper-operator/templates/roles.yaml index cccdd90a..a4dca793 100644 --- a/deploy/helm/zookeeper-operator/templates/roles.yaml +++ b/deploy/helm/zookeeper-operator/templates/roles.yaml @@ -68,6 +68,14 @@ rules: - customresourcedefinitions verbs: - get + - apiGroups: + - authentication.stackable.tech + resources: + - authenticationclasses + verbs: + - get + - list + - watch - apiGroups: - events.k8s.io resources: From dc393ed1531dd0e5a44698710f0e72e1a6f263be Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 11 May 2022 11:13:57 +0200 Subject: [PATCH 23/27] regenerated charts --- deploy/manifests/roles.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/manifests/roles.yaml b/deploy/manifests/roles.yaml index 8ff161da..616cda56 100644 --- a/deploy/manifests/roles.yaml +++ b/deploy/manifests/roles.yaml @@ -68,6 +68,14 @@ rules: - customresourcedefinitions verbs: - get + - apiGroups: + - authentication.stackable.tech + resources: + - authenticationclasses + verbs: + - get + - list + - watch - apiGroups: - events.k8s.io resources: From b5732a618325dcee5773ebb091134526f8ef7158 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 11 May 2022 13:04:21 +0200 Subject: [PATCH 24/27] Update deploy/config-spec/properties.yaml Co-authored-by: Sebastian Bernauer --- deploy/config-spec/properties.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index dff01dc0..20c23756 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -187,7 +187,7 @@ properties: - name: "server" required: false asOfVersion: "3.5.7" - description: "Specifies options to authenticate ssl connections from clients." + description: "Specifies options to authenticate ssl connections from other quorum members." - property: &sslQuorumHostNameVerification propertyNames: From 8ba1321184a5adf06dd3282e64588a4504d7c1d1 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 11 May 2022 13:29:27 +0200 Subject: [PATCH 25/27] adapted to pr review --- .../configs/properties.yaml | 2 +- deploy/manifests/configmap.yaml | 2 +- docs/modules/ROOT/pages/usage.adoc | 8 +- rust/operator-binary/src/command.rs | 6 +- rust/operator-binary/src/zk_controller.rs | 82 +++++++------------ 5 files changed, 41 insertions(+), 59 deletions(-) diff --git a/deploy/helm/zookeeper-operator/configs/properties.yaml b/deploy/helm/zookeeper-operator/configs/properties.yaml index dff01dc0..20c23756 100644 --- a/deploy/helm/zookeeper-operator/configs/properties.yaml +++ b/deploy/helm/zookeeper-operator/configs/properties.yaml @@ -187,7 +187,7 @@ properties: - name: "server" required: false asOfVersion: "3.5.7" - description: "Specifies options to authenticate ssl connections from clients." + description: "Specifies options to authenticate ssl connections from other quorum members." - property: &sslQuorumHostNameVerification propertyNames: diff --git a/deploy/manifests/configmap.yaml b/deploy/manifests/configmap.yaml index b4ec3711..2246ff29 100644 --- a/deploy/manifests/configmap.yaml +++ b/deploy/manifests/configmap.yaml @@ -191,7 +191,7 @@ data: - name: "server" required: false asOfVersion: "3.5.7" - description: "Specifies options to authenticate ssl connections from clients." + description: "Specifies options to authenticate ssl connections from other quorum members." - property: &sslQuorumHostNameVerification propertyNames: diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index c1abaf35..b0e1d7c7 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -47,18 +47,18 @@ Finally, a ConfigMap is created, containing a path that a ZooKeeper client can c The quorum and client communication are encrypted by default via TLS. This requires the https://github.com/stackabletech/secret-operator[Secret Operator] to be present in order to provide certificates. The utilized certificates can be changed in a top-level config. [source,yaml] -==== +---- include::example$example-cluster-tls-encryption.yaml[] -==== +---- <1> The `tls.secretClass` refers to the client-to-server encryption. Defaults to the `tls` secret. <2> The `quorumTlsSecretClass` refers to the server-to-server quorum encryption. Defaults to the `tls` secret. The `tls` secret is deployed from the https://github.com/stackabletech/secret-operator[Secret Operator] and looks like this: [source,yaml] -==== +---- include::example$example-secret-operator-tls-secret.yaml[] -==== +---- You can create your own secrets and reference them e.g. in the `tls.secretClass` to use different certificates. diff --git a/rust/operator-binary/src/command.rs b/rust/operator-binary/src/command.rs index f774b354..7187ba0a 100644 --- a/rust/operator-binary/src/command.rs +++ b/rust/operator-binary/src/command.rs @@ -62,11 +62,13 @@ pub fn create_init_container_command_args(zk: &ZookeeperCluster) -> String { args.join(" && ") } +/// Generates the shell script to retrieve a random 20 character password fn generate_password() -> String { - // taken from https://unix.stackexchange.com/questions/230673/how-to-generate-a-random-string format!("export {STORE_PASSWORD_ENV}=$(tr -dc A-Za-z0-9 String { format!( "echo {property}=${STORE_PASSWORD_ENV} >> {rwconf}/zoo.cfg", @@ -75,6 +77,8 @@ fn write_store_password_to_config(property: &str) -> String { ) } +/// Generates the shell script to create key and truststores from the certificates provided +/// by the secret operator fn create_key_and_trust_store_cmd(directory: &str) -> Vec { vec![ format!("echo [{dir}] Storing password", dir = directory), diff --git a/rust/operator-binary/src/zk_controller.rs b/rust/operator-binary/src/zk_controller.rs index 99294527..816d1281 100644 --- a/rust/operator-binary/src/zk_controller.rs +++ b/rust/operator-binary/src/zk_controller.rs @@ -15,18 +15,18 @@ use crate::{ }; use fnv::FnvHasher; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::commons::authentication::AuthenticationClassProvider; -use stackable_operator::k8s_openapi::api::core::v1::{CSIVolumeSource, EmptyDirVolumeSource}; -use stackable_operator::kube::api::DynamicObject; use stackable_operator::{ - builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, - commons::authentication::AuthenticationClass, + builder::{ + ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, + SecretOperatorVolumeSourceBuilder, VolumeBuilder, + }, + commons::authentication::{AuthenticationClass, AuthenticationClassProvider}, k8s_openapi::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, ConfigMapVolumeSource, EnvVar, EnvVarSource, ExecAction, - ObjectFieldSelector, PersistentVolumeClaim, PersistentVolumeClaimSpec, + ConfigMap, ConfigMapVolumeSource, EmptyDirVolumeSource, EnvVar, EnvVarSource, + ExecAction, ObjectFieldSelector, PersistentVolumeClaim, PersistentVolumeClaimSpec, PodSecurityContext, Probe, ResourceRequirements, SecurityContext, Service, ServicePort, ServiceSpec, Volume, }, @@ -34,7 +34,7 @@ use stackable_operator::{ apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector}, }, kube::{ - api::ObjectMeta, + api::{DynamicObject, ObjectMeta}, runtime::controller::{self, Context}, }, labels::{role_group_selector_labels, role_selector_labels}, @@ -570,28 +570,16 @@ fn build_server_rolegroup_statefulset( name: "rwconfig".to_string(), ..Volume::default() }) - .add_volume(Volume { - name: "quorum-tls".to_string(), - csi: Some(CSIVolumeSource { - driver: "secrets.stackable.tech".to_string(), - volume_attributes: Some( - vec![ - ( - "secrets.stackable.tech/class".to_string(), - zk.quorum_tls_secret_class(), - ), - ( - "secrets.stackable.tech/scope".to_string(), - "node,pod".to_string(), - ), - ] - .into_iter() - .collect::>(), - ), - ..CSIVolumeSource::default() - }), - ..Volume::default() - }) + .add_volume( + VolumeBuilder::new("quorum-tls") + .csi( + SecretOperatorVolumeSourceBuilder::new(zk.quorum_tls_secret_class()) + .with_node_scope() + .with_pod_scope() + .build(), + ) + .build(), + ) .security_context(PodSecurityContext { fs_group: Some(1000), ..PodSecurityContext::default() @@ -613,28 +601,18 @@ fn build_server_rolegroup_statefulset( None }; - pod_builder.add_volume(Volume { - name: "client-tls".to_string(), - csi: Some(CSIVolumeSource { - driver: "secrets.stackable.tech".to_string(), - volume_attributes: Some( - vec![ - ( - "secrets.stackable.tech/class".to_string(), - secret_class.unwrap_or_else(|| zk.client_tls_secret_class()), - ), - ( - "secrets.stackable.tech/scope".to_string(), - "node,pod".to_string(), - ), - ] - .into_iter() - .collect::>(), - ), - ..CSIVolumeSource::default() - }), - ..Volume::default() - }); + pod_builder.add_volume( + VolumeBuilder::new("client-tls") + .csi( + SecretOperatorVolumeSourceBuilder::new( + secret_class.unwrap_or_else(|| zk.client_tls_secret_class()), + ) + .with_node_scope() + .with_pod_scope() + .build(), + ) + .build(), + ); } let pod_builder = pod_builder.build_template(); From 925f59ffeedb6ee2d0d16d17c5f44039657beb7f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 11 May 2022 14:48:43 +0200 Subject: [PATCH 26/27] unset used env vars at the start --- tests/templates/kuttl/smoke/test_tls.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/templates/kuttl/smoke/test_tls.sh b/tests/templates/kuttl/smoke/test_tls.sh index 651dd5ec..1e80c21a 100755 --- a/tests/templates/kuttl/smoke/test_tls.sh +++ b/tests/templates/kuttl/smoke/test_tls.sh @@ -4,9 +4,13 @@ NAMESPACE=$1 SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2282" +# just to be safe... +unset CLIENT_STORE_SECRET +unset CLIENT_JVMFLAGS + echo "Start TLS testing..." ############################################################################ -# test the plaintext unsecured connection +# Test the plaintext unsecured connection ############################################################################ if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; then @@ -16,7 +20,7 @@ fi echo "[SUCCESS] Unsecure client connection established!" ############################################################################ -# we set the correct client tls credentials and expect to be able to connect +# We set the correct client tls credentials and expect to be able to connect ############################################################################ CLIENT_STORE_SECRET="$(< /stackable/rwconfig/zoo.cfg grep "ssl.keyStore.password" | cut -d "=" -f2)" export CLIENT_STORE_SECRET @@ -29,7 +33,6 @@ export CLIENT_JVMFLAGS=" -Dzookeeper.ssl.trustStore.location=/stackable/tls/client/truststore.p12 -Dzookeeper.ssl.trustStore.password=${CLIENT_STORE_SECRET}" - if ! /stackable/zookeeper/bin/zkCli.sh -server "${SERVER}" ls / &> /dev/null; then echo "[ERROR] Could not establish secure connection using client certificates!" From f91dda020b5dad6a8ee461ddfbc868b55a7ecdf5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 11 May 2022 16:10:30 +0200 Subject: [PATCH 27/27] added unset for QUORUM_STORE_SECRET --- tests/templates/kuttl/smoke/test_tls.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/templates/kuttl/smoke/test_tls.sh b/tests/templates/kuttl/smoke/test_tls.sh index 1e80c21a..bf68c882 100755 --- a/tests/templates/kuttl/smoke/test_tls.sh +++ b/tests/templates/kuttl/smoke/test_tls.sh @@ -5,6 +5,7 @@ NAMESPACE=$1 SERVER="test-zk-server-primary-1.test-zk-server-primary.${NAMESPACE}.svc.cluster.local:2282" # just to be safe... +unset QUORUM_STORE_SECRET unset CLIENT_STORE_SECRET unset CLIENT_JVMFLAGS