From 7842e25c19f1e129f2bb8dfbac4eb2c6bb52a44f Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 22 Jul 2021 17:57:53 +0200 Subject: [PATCH 1/2] Set KUBECONFIG in systemd services The environment variable KUBECONFIG is set in the systemd services. It contains the path to the kubeconfig file which is also used by the Stackable agent. --- Cargo.lock | 5 ++ Cargo.toml | 2 + src/provider/states/pod/creating_service.rs | 40 ++++++++++++- src/provider/systemdmanager/systemdunit.rs | 65 +++++++++++++-------- 4 files changed, 86 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6baa8209..92304ec1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1627,6 +1627,9 @@ name = "multimap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +dependencies = [ + "serde", +] [[package]] name = "multipart" @@ -2749,6 +2752,7 @@ dependencies = [ "anyhow", "async-trait", "byteorder", + "dirs", "env_logger 0.9.0", "flate2", "futures-util", @@ -2761,6 +2765,7 @@ dependencies = [ "kubelet", "lazy_static", "log", + "multimap", "nix 0.22.0", "oci-distribution", "regex", diff --git a/Cargo.toml b/Cargo.toml index ae65a140..2fcb47c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ version = "0.5.0-nightly" anyhow = "1.0" async-trait = "0.1" byteorder = "1.4" +dirs = "3.0" env_logger = "0.9" flate2 = "1.0" futures-util = "0.3" @@ -25,6 +26,7 @@ kubelet = { git = "https://github.com/stackabletech/krustlet.git", branch = "sta Inflector = "0.11" lazy_static = "1.4" log = "0.4" +multimap = "0.8" nix = "0.22" oci-distribution = { git = "https://github.com/stackabletech/krustlet.git", branch = "stackable_patches_v0.7.0" } # version = "0.6" regex = "1.4" diff --git a/src/provider/states/pod/creating_service.rs b/src/provider/states/pod/creating_service.rs index 9c632c67..6e783d1c 100644 --- a/src/provider/states/pod/creating_service.rs +++ b/src/provider/states/pod/creating_service.rs @@ -3,14 +3,17 @@ use kubelet::{ container::ContainerKey, pod::{Pod, PodKey}, }; -use log::{debug, error, info}; +use log::{debug, error, info, warn}; use super::setup_failed::SetupFailed; use super::starting::Starting; use crate::provider::systemdmanager::systemdunit::SystemDUnit; use crate::provider::{ContainerHandle, PodState, ProviderState}; use anyhow::Error; +use dirs::home_dir; +use std::env; use std::fs::create_dir_all; +use std::path::PathBuf; #[derive(Default, Debug, TransitionTo)] #[transition_to(Starting, SetupFailed)] @@ -72,7 +75,7 @@ impl State for CreatingService { // systemd unit file/service. // Map every container from the pod object to a systemdunit for container in &pod.containers() { - let unit = match SystemDUnit::new( + let mut unit = match SystemDUnit::new( &unit_template, &service_prefix, &container, @@ -83,6 +86,30 @@ impl State for CreatingService { Err(err) => return Transition::Complete(Err(Error::from(err))), }; + if let Some(kubeconfig_path) = find_kubeconfig() { + const UNIT_ENV_KEY: &str = "KUBECONFIG"; + if let Some(kubeconfig_path) = kubeconfig_path.to_str() { + unit.add_env_var(UNIT_ENV_KEY, kubeconfig_path); + } else { + warn!( + "Environment variable {} cannot be added to \ + the systemd service [{}] because the path [{}] \ + is not valid unicode.", + UNIT_ENV_KEY, + service_name, + kubeconfig_path.to_string_lossy() + ); + } + } else { + warn!( + "Kubeconfig file not found. It will not be added \ + to the environment variables of the systemd \ + service [{}]. If no kubeconfig is present then the \ + Stackable agent should have generated one.", + service_name + ); + } + // Create the service // As per ADR005 we currently write the unit files directly in the systemd // unit directory (by passing None as [unit_file_path]). @@ -121,3 +148,12 @@ impl State for CreatingService { Ok(make_status(Phase::Pending, &"CreatingService")) } } + +/// Tries to find the kubeconfig file in the environment variable +/// `KUBECONFIG` and on the path `$HOME/.kube/config` +fn find_kubeconfig() -> Option { + let env_var = env::var_os("KUBECONFIG").map(PathBuf::from); + let default_path = || home_dir().map(|home| home.join(".kube").join("config")); + + env_var.or_else(default_path).filter(|path| path.exists()) +} diff --git a/src/provider/systemdmanager/systemdunit.rs b/src/provider/systemdmanager/systemdunit.rs index a82f7814..663d6fc1 100644 --- a/src/provider/systemdmanager/systemdunit.rs +++ b/src/provider/systemdmanager/systemdunit.rs @@ -11,10 +11,11 @@ use crate::provider::states::pod::PodState; use crate::provider::systemdmanager::manager::UnitTypes; use lazy_static::lazy_static; use log::{debug, error, info, trace, warn}; +use multimap::MultiMap; use regex::Regex; use std::fmt; use std::fmt::{Display, Formatter}; -use std::iter; +use std::iter::{self, repeat}; use strum::{Display, EnumIter, IntoEnumIterator}; /// The default timeout for stopping a service, after this has passed systemd will terminate @@ -43,7 +44,7 @@ lazy_static! { pub struct SystemDUnit { pub name: String, pub unit_type: UnitTypes, - pub sections: HashMap>, + pub sections: HashMap>, } // TODO: The parsing code is also highly stackable specific, we should @@ -106,41 +107,35 @@ impl SystemDUnit { unit.name = format!("{}{}", name_prefix, trimmed_name); - unit.add_property(Section::Unit, "Description", &unit.name.clone()); + unit.set_property(Section::Unit, "Description", &unit.name.clone()); - unit.add_property( + unit.set_property( Section::Service, "ExecStart", &SystemDUnit::get_command(container, template_data, package_root)?, ); let env_vars = SystemDUnit::get_environment(container, service_name, template_data)?; - if !env_vars.is_empty() { - let mut assignments = env_vars - .iter() - .map(|(k, v)| format!("\"{}={}\"", k, v)) - .collect::>(); - assignments.sort(); - // TODO Put every environment variable on a separate line - unit.add_property(Section::Service, "Environment", &assignments.join(" ")); + for (key, value) in env_vars { + unit.add_env_var(&key, &value); } // These are currently hard-coded, as this is not something we expect to change soon - unit.add_property(Section::Service, "StandardOutput", "journal"); - unit.add_property(Section::Service, "StandardError", "journal"); + unit.set_property(Section::Service, "StandardOutput", "journal"); + unit.set_property(Section::Service, "StandardError", "journal"); if let Some(user_name) = SystemDUnit::get_user_name_from_security_context(container, &unit.name)? { if !user_mode { - unit.add_property(Section::Service, "User", user_name); + unit.set_property(Section::Service, "User", user_name); } else { info!("The user name [{}] in spec.containers[name = {}].securityContext.windowsOptions.runAsUserName is not set in the systemd unit because the agent runs in session mode.", user_name, container.name()); } } // This one is mandatory, as otherwise enabling the unit fails - unit.add_property(Section::Install, "WantedBy", "multi-user.target"); + unit.set_property(Section::Install, "WantedBy", "multi-user.target"); Ok(unit) } @@ -208,10 +203,10 @@ impl SystemDUnit { } .to_string(); - unit.add_property(Section::Service, "TimeoutStopSec", &termination_timeout); + unit.set_property(Section::Service, "TimeoutStopSec", &termination_timeout); if let Some(stop_timeout) = pod_spec.termination_grace_period_seconds { - unit.add_property( + unit.set_property( Section::Service, "TimeoutStopSec", stop_timeout.to_string().as_str(), @@ -220,7 +215,7 @@ impl SystemDUnit { if let Some(user_name) = SystemDUnit::get_user_name_from_pod_security_context(pod)? { if !user_mode { - unit.add_property(Section::Service, "User", user_name); + unit.set_property(Section::Service, "User", user_name); } else { info!("The user name [{}] in spec.securityContext.windowsOptions.runAsUserName is not set in the systemd unit because the agent runs in session mode.", user_name); } @@ -262,9 +257,29 @@ impl SystemDUnit { format!("{}.{}", self.name, lower_type) } - /// Add a key=value entry to the specified section + /// Adds an environment variable to the service section of the unit file + pub fn add_env_var(&mut self, key: &str, value: &str) { + self.add_property( + Section::Service, + "Environment", + &format!("\"{}={}\"", key, value), + ); + } + + /// Sets a property in the given section + /// + /// If properties with the given key already exist then they are + /// replaced with the given one. + fn set_property(&mut self, section: Section, key: &str, value: &str) { + let section = self.sections.entry(section).or_default(); + *section.entry(String::from(key)).or_insert_vec(Vec::new()) = vec![String::from(value)]; + } + + /// Adds a property to the given section + /// + /// Properties with the same key remain untouched. fn add_property(&mut self, section: Section, key: &str, value: &str) { - let section = self.sections.entry(section).or_insert_with(HashMap::new); + let section = self.sections.entry(section).or_default(); section.insert(String::from(key), String::from(value)); } @@ -278,11 +293,12 @@ impl SystemDUnit { .join("\n\n") } - fn write_section(section: &Section, entries: &HashMap) -> String { + fn write_section(section: &Section, entries: &MultiMap) -> String { let header = format!("[{}]", section); let mut body = entries - .iter() + .iter_all() + .flat_map(|(key, values)| repeat(key).zip(values)) .map(|(key, value)| format!("{}={}", key, value)) .collect::>(); body.sort(); @@ -523,7 +539,8 @@ mod test { Description=default-stackable-test-container [Service] - Environment="LOG_DIR=/var/log/default-stackable" "LOG_LEVEL=INFO" + Environment="LOG_DIR=/var/log/default-stackable" + Environment="LOG_LEVEL=INFO" ExecStart=start.sh arg /etc/default-stackable StandardError=journal StandardOutput=journal From 4351fb86bf708488bc027010ce51d9a885504f3d Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 23 Jul 2021 11:05:38 +0200 Subject: [PATCH 2/2] Changelog updated --- CHANGELOG.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 0f6bcf76..a7379c75 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -4,9 +4,11 @@ :224: https://github.com/stackabletech/agent/pull/224[#224] :229: https://github.com/stackabletech/agent/pull/229[#229] +:234: https://github.com/stackabletech/agent/pull/234[#234] === Added * `hostIP` and `podIP` added to the pod status ({224}). +* Environment variable `KUBECONFIG` set in systemd services ({234}). === Fixed * Invalid or unreachable repositories are skipped when searching for