From 53cb1eb853821567d9d20456bb091f80206a1abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 17 Feb 2021 18:00:01 +0100 Subject: [PATCH 1/4] Changed dependency on Krustlet to the Stackable fork which skips pod eviction on shutdown. --- Cargo.lock | 6 +++--- Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1da201..5c7f6a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1259,7 +1259,7 @@ dependencies = [ [[package]] name = "kubelet" version = "0.5.0" -source = "git+https://github.com/deislabs/krustlet.git?rev=ac218b38ba564de806568e49d9e38aaef9f41537#ac218b38ba564de806568e49d9e38aaef9f41537" +source = "git+https://github.com/stackabletech/krustlet.git?rev=bb8bb42c9400a565df4be04f357e61934fb277c6#bb8bb42c9400a565df4be04f357e61934fb277c6" dependencies = [ "anyhow", "async-stream 0.3.0", @@ -1308,7 +1308,7 @@ dependencies = [ [[package]] name = "kubelet-derive" version = "0.1.0" -source = "git+https://github.com/deislabs/krustlet.git?rev=ac218b38ba564de806568e49d9e38aaef9f41537#ac218b38ba564de806568e49d9e38aaef9f41537" +source = "git+https://github.com/stackabletech/krustlet.git?rev=bb8bb42c9400a565df4be04f357e61934fb277c6#bb8bb42c9400a565df4be04f357e61934fb277c6" dependencies = [ "quote 1.0.7", "syn 1.0.50", @@ -1586,7 +1586,7 @@ dependencies = [ [[package]] name = "oci-distribution" version = "0.4.0" -source = "git+https://github.com/deislabs/krustlet.git?rev=ac218b38ba564de806568e49d9e38aaef9f41537#ac218b38ba564de806568e49d9e38aaef9f41537" +source = "git+https://github.com/stackabletech/krustlet.git?rev=bb8bb42c9400a565df4be04f357e61934fb277c6#bb8bb42c9400a565df4be04f357e61934fb277c6" dependencies = [ "anyhow", "futures-util", diff --git a/Cargo.toml b/Cargo.toml index a38b694..693da46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,8 +8,8 @@ edition = "2018" # We are currently referencing the Krustlet directly from the repository, because some features that we are using have # (exponential backoff most prominently) have not yet been included in a release # We will look to move this to officially released versions as soon as possible -kubelet = { git="https://github.com/deislabs/krustlet.git", rev="ac218b38ba564de806568e49d9e38aaef9f41537", default-features = true, features= ["derive", "cli"] } -oci-distribution = { git="https://github.com/deislabs/krustlet.git", rev="ac218b38ba564de806568e49d9e38aaef9f41537"} +kubelet = { git="https://github.com/stackabletech/krustlet.git", rev="bb8bb42c9400a565df4be04f357e61934fb277c6", default-features = true, features= ["derive", "cli"] } +oci-distribution = { git="https://github.com/stackabletech/krustlet.git", rev="bb8bb42c9400a565df4be04f357e61934fb277c6"} k8s-openapi = { version = "0.9", default-features = false, features = ["v1_18"] } kube = { version= "0.42", default-features = false, features = ["native-tls"] } kube-derive = "0.43" From 54f1b9875da47ec07780933857e1200ee6dadf70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Thu, 18 Feb 2021 13:34:44 +0100 Subject: [PATCH 2/4] Change dependency on Krustlet to use our fork, which skips draining pods when shutting down. Draining all pods on agent shudown would mean stopping all services that are managed by the agent, which is not feasible for our scenario. Also added check during startup whether the service is already up and running, if it is already running we don't start it again. --- src/provider/states/starting.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/provider/states/starting.rs b/src/provider/states/starting.rs index 5f32803..8d64759 100644 --- a/src/provider/states/starting.rs +++ b/src/provider/states/starting.rs @@ -20,6 +20,28 @@ impl State for Starting { async fn next(self: Box, pod_state: &mut PodState, _: &Pod) -> Transition { if let Some(systemd_units) = &pod_state.service_units { for unit in systemd_units { + match pod_state.systemd_manager.is_running(&unit.get_name()) { + Ok(true) => { + debug!( + "Unit [{}] for service [{}] already running, nothing to do..", + &unit.get_name(), + &pod_state.service_name + ); + // Skip rest of loop as the service is already running + continue; + } + Err(dbus_error) => { + debug!( + "Error retrieving activestate of unit [{}] for service [{}]: [{}]", + &unit.get_name(), + &pod_state.service_name, + dbus_error + ); + return Transition::Complete(Err(anyhow::Error::from(dbus_error))); + } + _ => { // nothing to do, just keep going + } + } info!("Starting systemd unit [{}]", unit); if let Err(start_error) = pod_state.systemd_manager.start(&unit.get_name()) { error!( From d05cba850acefab0c15682784faee1384073c5af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Thu, 18 Feb 2021 13:35:40 +0100 Subject: [PATCH 3/4] Fixed clippy warning --- src/provider/states/starting.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/provider/states/starting.rs b/src/provider/states/starting.rs index 8d64759..ecaba5a 100644 --- a/src/provider/states/starting.rs +++ b/src/provider/states/starting.rs @@ -37,7 +37,7 @@ impl State for Starting { &pod_state.service_name, dbus_error ); - return Transition::Complete(Err(anyhow::Error::from(dbus_error))); + return Transition::Complete(Err(dbus_error)); } _ => { // nothing to do, just keep going } From a7c93afa2b64d7669d50ec0e53a95d36429c6a9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Thu, 18 Feb 2021 22:28:27 +0100 Subject: [PATCH 4/4] Updated comment in cargo.toml on Krustlet dependency --- Cargo.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 979aae4..62b1bc0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,9 +5,12 @@ authors = ["Sönke Liebau "] edition = "2018" [dependencies] -# We are currently referencing the Krustlet directly from the repository, because some features that we are using have -# (exponential backoff most prominently) have not yet been included in a release -# We will look to move this to officially released versions as soon as possible +# We are currently referencing the Krustlet directly from a Stackable fork of the official repository. +# There are two reasons for this, the fork is needed to remove the node draining behavior of the Krustlet (see +# https://github.com/deislabs/krustlet/issues/523) +# Our fork / the referenced commit is quite a bit behind the official repo, as there were some breaking changes in the +# 0.6 release that we need to adapt the Krustlet to. +# We will look to move to the latest version as soon as possible, but may need to continue releasing from a fork. kubelet = { git="https://github.com/stackabletech/krustlet.git", rev="bb8bb42c9400a565df4be04f357e61934fb277c6", default-features = true, features= ["derive", "cli"] } oci-distribution = { git="https://github.com/stackabletech/krustlet.git", rev="bb8bb42c9400a565df4be04f357e61934fb277c6"} k8s-openapi = { version = "0.9", default-features = false, features = ["v1_18"] }