diff --git a/Cargo.lock b/Cargo.lock index dd2f8d9..c41363a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2574,6 +2574,7 @@ dependencies = [ "env_logger", "flate2", "handlebars", + "hostname", "indoc", "k8s-openapi", "kube", diff --git a/Cargo.toml b/Cargo.toml index 243a11d..0c0ce56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ edition = "2018" # We will look to move this to officially released versions as soon as possible kubelet = { git="https://github.com/deislabs/krustlet.git", rev="ac218b38ba564de806568e49d9e38aaef9f41537", default-features = true, features= ["derive", "cli"] } oci-distribution = { git="https://github.com/deislabs/krustlet.git", rev="ac218b38ba564de806568e49d9e38aaef9f41537"} - k8s-openapi = { version = "0.9", default-features = false, features = ["v1_18"] } kube = { version= "0.42", default-features = false, features = ["native-tls"] } kube-derive = "0.43" @@ -30,6 +29,7 @@ thiserror = "1.0" url = "2.2" pnet = "0.26.0" stackable_config = { git = "https://github.com/stackabletech/common.git", branch = "main" } +hostname = "0.3" [dev-dependencies] indoc = "1.0" diff --git a/src/agentconfig.rs b/src/agentconfig.rs index e455176..66aa119 100644 --- a/src/agentconfig.rs +++ b/src/agentconfig.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use std::collections::hash_map::RandomState; use std::collections::{HashMap, HashSet}; use std::net::IpAddr; @@ -9,33 +10,70 @@ use pnet::datalink; use stackable_config::{ConfigOption, Configurable, Configuration}; use thiserror::Error; -use crate::agentconfig::AgentConfigError::WrongArgumentCount; +use crate::agentconfig::AgentConfigError::{ArgumentParseError, WrongArgumentCount}; #[derive(Error, Debug)] pub enum AgentConfigError { #[error("Wrong number of arguments found for config option {}!", .option.name)] WrongArgumentCount { option: ConfigOption }, + #[error("Unable to parse value for parameter [{}]!", .name)] + ArgumentParseError { name: String }, } #[derive(Clone)] pub struct AgentConfig { + pub hostname: String, pub parcel_directory: PathBuf, pub config_directory: PathBuf, pub log_directory: PathBuf, + pub bootstrap_file: PathBuf, + pub data_directory: PathBuf, pub server_ip_address: IpAddr, + pub server_port: u16, pub server_cert_file: Option, pub server_key_file: Option, pub tags: HashMap, } impl AgentConfig { + pub const HOSTNAME: ConfigOption = ConfigOption { + name: "hostname", + default: None, + required: false, + takes_argument: true, + help: + "The hostname to register the node under in Kubernetes - defaults to system hostname.", + documentation: include_str!("config_documentation/hostname.adoc"), + list: false, + }; + + pub const DATA_DIR: ConfigOption = ConfigOption { + name: "data-directory", + default: Some("/var/stackable/agent/data"), + required: false, + takes_argument: true, + help: "The directory where the stackable agent should keep its working data.", + documentation: include_str!("config_documentation/data_directory.adoc"), + list: false, + }; + + pub const BOOTSTRAP_FILE: ConfigOption = ConfigOption { + name: "bootstrap-file", + default: Some("/etc/kubernetes/bootstrap-kubelet.conf"), + required: false, + takes_argument: true, + help: "The bootstrap file to use in case Kubernetes bootstraping is used to add the agent.", + documentation: include_str!("config_documentation/bootstrap_file.adoc"), + list: false, + }; + pub const SERVER_IP_ADDRESS: ConfigOption = ConfigOption { name: "server-bind-ip", default: None, required: false, takes_argument: true, help: "The local IP to register as the node's ip with the apiserver. Will be automatically set to the first address of the first non-loopback interface if not specified.", - documentation: "", + documentation: include_str!("config_documentation/server_ip_address.adoc"), list: false, }; @@ -45,7 +83,7 @@ impl AgentConfig { required: false, takes_argument: true, help: "The certificate file for the local webserver which the Krustlet starts.", - documentation: "", + documentation: include_str!("config_documentation/server_cert_file.adoc"), list: false, }; @@ -56,7 +94,17 @@ impl AgentConfig { takes_argument: true, help: "Private key file (in PKCS8 format) to use for the local webserver the Krustlet starts.", - documentation: "", + documentation: include_str!("config_documentation/server_key_file.adoc"), + list: false, + }; + + pub const SERVER_PORT: ConfigOption = ConfigOption { + name: "server-port", + default: Some("3000"), + required: false, + takes_argument: true, + help: "Port to listen on for callbacks.", + documentation: include_str!("config_documentation/server_port.adoc"), list: false, }; @@ -66,12 +114,7 @@ impl AgentConfig { required: false, takes_argument: true, help: "The base directory under which installed packages will be stored.", - documentation: "This directory will serve as starting point for packages that are needed by \ - pods assigned to this node.\n Packages will be downloaded into the \"_download\" folder at the -top level of this folder as archives and remain there for potential future use.\n\ - Archives will the be extracted directly into this folder in subdirectories following the naming -scheme of \"productname-productversion\". - The agent will need full access to this directory and tries to create it if it does not exist.", + documentation: include_str!("config_documentation/package_directory.adoc"), list: false, }; @@ -81,12 +124,7 @@ scheme of \"productname-productversion\". required: false, takes_argument: true, help: "The base directory under which configuration will be generated for all executed services.", - documentation: "This directory will serve as starting point for all log files which this service creates.\ - Every service will get its own subdirectories created within this directory - for every service start a \ - new subdirectory will be created to show a full history of configuration that was used for this service.\n - ConfigMaps that are mounted into the pod that describes this service will be created relative to these run \ - directories - unless the mounts specify an absolute path, in which case it is allowed to break out of this directory.\n\n\ - The agent will need full access to this directory and tries to create it if it does not exist.", + documentation: include_str!("config_documentation/config_directory.adoc"), list: false, }; @@ -96,10 +134,7 @@ scheme of \"productname-productversion\". required: false, takes_argument: true, help: "The base directory under which log files will be placed for all services.", - documentation: "This directory will serve as starting point for all log files which this service creates.\ - Every service will get its own subdirectory created within this directory.\n - Anything that is then specified in the log4j config or similar files will be resolved relatively to this directory.\n\n\ - The agent will need full access to this directory and tries to create it if it does not exist.", + documentation: include_str!("config_documentation/log_directory.adoc"), list: false, }; @@ -109,7 +144,7 @@ scheme of \"productname-productversion\". required: false, takes_argument: false, help: "If this option is specified, any file referenced in AGENT_CONF environment variable will be ignored.", - documentation: "", + documentation: include_str!("config_documentation/no_config.adoc"), list: false, }; @@ -119,20 +154,24 @@ scheme of \"productname-productversion\". required: false, takes_argument: true, help: "A \"key=value\" pair that should be assigned to this agent as tag. This can be specified multiple times to assign additional tags.", - documentation: "Tags are the main way of identifying nodes to assign services to later on.", + documentation: include_str!("config_documentation/tags.adoc"), list: true }; fn get_options() -> HashSet { [ + AgentConfig::HOSTNAME, + AgentConfig::DATA_DIR, AgentConfig::SERVER_IP_ADDRESS, AgentConfig::SERVER_CERT_FILE, AgentConfig::SERVER_KEY_FILE, + AgentConfig::SERVER_PORT, AgentConfig::PACKAGE_DIR, AgentConfig::CONFIG_DIR, AgentConfig::LOG_DIR, AgentConfig::NO_CONFIG, AgentConfig::TAG, + AgentConfig::BOOTSTRAP_FILE, ] .iter() .cloned() @@ -161,12 +200,12 @@ scheme of \"productname-productversion\". parsed_values.get(option) ); if let Some(Some(list_value)) = parsed_values.get(option) { - if list_value.len() != 1 { - error!("Got additional, unexpected values for parameter!"); - } else { + if list_value.len() == 1 { // We've checked that the list has exactly one value at this point, so no errors should // occur after this point - but you never know return Ok(list_value[0].to_string()); + } else { + error!("Got additional, unexpected values for parameter!"); } } Err(WrongArgumentCount { @@ -174,6 +213,38 @@ scheme of \"productname-productversion\". }) } + /// Helper method to retrieve a path from the config and convert this to a PathBuf directly. + /// This method assumes that a default value has been specified for this option and panics if + /// no value can be retrieved (should only happen if assigning the default value fails or + /// one was not specified) + /// + /// # Panics + /// This function panics if the parsed_values object does not contain a value for the key. + /// This is due to the fact that we expect a default value to be defined for these parameters, + /// so if we do not get a value that default value has not been defined or something else went + /// badly wrong. + fn get_with_default( + parsed_values: &HashMap>>, + option: &ConfigOption, + error_list: &mut Vec, + ) -> Result { + T::from_str( + &AgentConfig::get_exactly_one_string(&parsed_values, option).unwrap_or_else(|_| { + panic!( + "No value present for parameter {} even though it should have a default value!", + option.name + ) + }), + ) + .map_err(|_| { + let error = ArgumentParseError { + name: option.name.to_string(), + }; + error_list.push(error); + anyhow!("Error for parameter: {}", option.name) + }) + } + /// This tries to find the first non loopback interface with an ip address assigned. /// This should usually be the default interface according to: /// @@ -201,6 +272,12 @@ scheme of \"productname-productversion\". }; None } + + fn default_hostname() -> anyhow::Result { + hostname::get()? + .into_string() + .map_err(|_| anyhow::anyhow!("invalid utf-8 hostname string")) + } } impl Configurable for AgentConfig { @@ -216,6 +293,14 @@ impl Configurable for AgentConfig { fn parse_values( parsed_values: HashMap>, RandomState>, ) -> Result { + // Parse hostname or lookup local hostname + let final_hostname = + AgentConfig::get_exactly_one_string(&parsed_values, &AgentConfig::HOSTNAME) + .unwrap_or_else(|_| { + AgentConfig::default_hostname() + .unwrap_or_else(|_| panic!("Unable to get hostname!")) + }); + // Parse IP Address or lookup default let final_ip = if let Ok(ip) = AgentConfig::get_exactly_one_string(&parsed_values, &AgentConfig::SERVER_IP_ADDRESS) @@ -228,59 +313,46 @@ impl Configurable for AgentConfig { }; info!("Selected {} as local address to listen on.", final_ip); + let mut error_list = vec![]; + + // Parse directory/file parameters + // PathBuf::from_str returns an infallible as Error, so cannot fail, hence unwrap is save + // to use for PathBufs here + + // Parse data directory from values, add any error that occured to the list of errors + let final_data_dir = AgentConfig::get_with_default( + &parsed_values, + &AgentConfig::DATA_DIR, + error_list.as_mut(), + ); + + // Parse bootstrap file from values + let final_bootstrap_file = AgentConfig::get_with_default( + &parsed_values, + &AgentConfig::BOOTSTRAP_FILE, + error_list.as_mut(), + ); + // Parse log directory - let final_log_dir = if let Ok(log_dir) = - AgentConfig::get_exactly_one_string(&parsed_values, &AgentConfig::LOG_DIR) - { - PathBuf::from_str(&log_dir).unwrap_or_else(|_| { - panic!("Error parsing valid log directory from string: {}", log_dir) - }) - } else { - PathBuf::from_str( - AgentConfig::LOG_DIR - .default - .expect("Invalid default value for log directory option!"), - ) - .unwrap_or_else(|_| panic!("Unable to get log directory from options!".to_string())) - }; + let final_log_dir = AgentConfig::get_with_default( + &parsed_values, + &AgentConfig::LOG_DIR, + error_list.as_mut(), + ); // Parse config directory - let final_config_dir = if let Ok(config_dir) = - AgentConfig::get_exactly_one_string(&parsed_values, &AgentConfig::CONFIG_DIR) - { - PathBuf::from_str(&config_dir).unwrap_or_else(|_| { - panic!( - "Error parsing valid config directory from string: {}", - config_dir - ) - }) - } else { - PathBuf::from_str( - AgentConfig::CONFIG_DIR - .default - .expect("Invalid default value for config directory option!"), - ) - .unwrap_or_else(|_| panic!("Unable to get config directory from options!".to_string())) - }; + let final_config_dir = AgentConfig::get_with_default( + &parsed_values, + &AgentConfig::CONFIG_DIR, + error_list.as_mut(), + ); // Parse parcel directory - let final_parcel_dir = if let Ok(parcel_dir) = - AgentConfig::get_exactly_one_string(&parsed_values, &AgentConfig::PACKAGE_DIR) - { - PathBuf::from_str(&parcel_dir).unwrap_or_else(|_| { - panic!( - "Error parsing valid parcel directory from string: {}", - parcel_dir - ) - }) - } else { - PathBuf::from_str( - AgentConfig::PACKAGE_DIR - .default - .expect("Invalid default value for parcel directory option!"), - ) - .unwrap_or_else(|_| panic!("Unable to get parcel directory from options!".to_string())) - }; + let final_package_dir = AgentConfig::get_with_default( + &parsed_values, + &AgentConfig::PACKAGE_DIR, + error_list.as_mut(), + ); // Parse cert file let final_server_cert_file = if let Ok(server_cert_file) = @@ -310,30 +382,53 @@ impl Configurable for AgentConfig { None }; + let final_port = AgentConfig::get_with_default( + &parsed_values, + &AgentConfig::SERVER_PORT, + error_list.as_mut(), + ); + let mut final_tags: HashMap = HashMap::new(); if let Some(Some(tags)) = parsed_values.get(&AgentConfig::TAG) { for tag in tags { let split: Vec<&str> = tag.split('=').collect(); - if split.len() != 2 { + if split.len() == 2 { + // This might panic, but really shouldn't, as we've checked the size of the array + final_tags.insert(split[0].to_string(), split[1].to_string()); + } else { // We want to avoid any "unpredictable" behavior like ignoring a malformed // key=value pair with just a log message -> so we panic if this can't be // parsed - panic!(format!( - "Unable to parse value [{}] for option --tag as key=value pair!", - tag - )) - } else { - // This might panic, but really shouldn't, as we've checked the size of the array - final_tags.insert(split[0].to_string(), split[1].to_string()); + error_list.push(ArgumentParseError { + name: AgentConfig::TAG.name.to_string(), + }); } } } + // Panic if we encountered any errors during parsing of the values + if !error_list.is_empty() { + panic!( + "Error parsing command line parameters:\n{}", + error_list + .into_iter() + .map(|thiserror| format!("{:?}\n", thiserror)) + .collect::() + ); + } + + // These unwraps are ok to panic, if one of them barfs then something went horribly wrong + // above, as we should have paniced in a "controlled fashion" from the conditional block + // right before this Ok(AgentConfig { - parcel_directory: final_parcel_dir, - config_directory: final_config_dir, - log_directory: final_log_dir, + hostname: final_hostname, + parcel_directory: final_package_dir.unwrap(), + config_directory: final_config_dir.unwrap(), + data_directory: final_data_dir.unwrap(), + log_directory: final_log_dir.unwrap(), + bootstrap_file: final_bootstrap_file.unwrap(), server_ip_address: final_ip, + server_port: final_port.unwrap(), server_cert_file: final_server_cert_file, server_key_file: final_server_key_file, tags: final_tags, diff --git a/src/config_documentation/bootstrap_file.adoc b/src/config_documentation/bootstrap_file.adoc new file mode 100644 index 0000000..e69de29 diff --git a/src/config_documentation/config_directory.adoc b/src/config_documentation/config_directory.adoc new file mode 100644 index 0000000..0551dbf --- /dev/null +++ b/src/config_documentation/config_directory.adoc @@ -0,0 +1,11 @@ +This directory will serve as starting point for all log files which this service creates. + +Every service will get its own subdirectories created within this directory - for every service start a +new subdirectory will be created to show a full history of configuration that was used for this service. + +ConfigMaps which are specified in the pod that describes this service will be created relative to these run +directories - unless the mounts specify an absolute path, in which case it is allowed to break out of this directory. + +WARNING: This allows anybody who can specify pods more or less full access to the file system on the machine running the agent! + +The agent will need full access to this directory and tries to create it if it does not exist. \ No newline at end of file diff --git a/src/config_documentation/data_directory.adoc b/src/config_documentation/data_directory.adoc new file mode 100644 index 0000000..e69de29 diff --git a/src/config_documentation/hostname.adoc b/src/config_documentation/hostname.adoc new file mode 100644 index 0000000..e69de29 diff --git a/src/config_documentation/log_directory.adoc b/src/config_documentation/log_directory.adoc new file mode 100644 index 0000000..10264a1 --- /dev/null +++ b/src/config_documentation/log_directory.adoc @@ -0,0 +1,5 @@ +This directory will serve as starting point for all log files which this service creates. +Every service will get its own subdirectory created within this directory. +Anything that is then specified in the log4j config or similar files will be resolved relatively to this directory. + +The agent will need full access to this directory and tries to create it if it does not exist. \ No newline at end of file diff --git a/src/config_documentation/no_config.adoc b/src/config_documentation/no_config.adoc new file mode 100644 index 0000000..e69de29 diff --git a/src/config_documentation/package_directory.adoc b/src/config_documentation/package_directory.adoc new file mode 100644 index 0000000..b619ac3 --- /dev/null +++ b/src/config_documentation/package_directory.adoc @@ -0,0 +1,6 @@ +This directory will serve as starting point for packages that are needed by pods assigned to this node.\n Packages will be downloaded into the "_download" folder at the top level of this folder as archives and remain there for potential future use. + +Archives will the be extracted directly into this folder in subdirectories following the naming +scheme of "productname-productversion". + +The agent will need full access to this directory and tries to create it if it does not exist. \ No newline at end of file diff --git a/src/config_documentation/plugin_directory.adoc b/src/config_documentation/plugin_directory.adoc new file mode 100644 index 0000000..e69de29 diff --git a/src/config_documentation/server_cert_file.adoc b/src/config_documentation/server_cert_file.adoc new file mode 100644 index 0000000..e69de29 diff --git a/src/config_documentation/server_ip_address.adoc b/src/config_documentation/server_ip_address.adoc new file mode 100644 index 0000000..e69de29 diff --git a/src/config_documentation/server_key_file.adoc b/src/config_documentation/server_key_file.adoc new file mode 100644 index 0000000..e69de29 diff --git a/src/config_documentation/server_port.adoc b/src/config_documentation/server_port.adoc new file mode 100644 index 0000000..e69de29 diff --git a/src/config_documentation/tags.adoc b/src/config_documentation/tags.adoc new file mode 100644 index 0000000..8d814eb --- /dev/null +++ b/src/config_documentation/tags.adoc @@ -0,0 +1,3 @@ +A "key=value" pair that should be assigned to this agent as tag. This can be specified multiple times to assign additional tags. + +Tags are the main way of identifying nodes to assign services to later on. \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 5a3bd37..2d3d0d1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,7 @@ use std::ffi::OsString; use kube::config::Config as KubeConfig; use kube::config::KubeConfigOptions; -use kubelet::config::Config; +use kubelet::config::{Config, ServerConfig}; use kubelet::Kubelet; use log::{info, warn}; use stackable_config::ConfigBuilder; @@ -46,19 +46,40 @@ async fn main() -> anyhow::Result<()> { export_env("NODE_LABELS", &node_labels); - if let Some(cert_file_path) = agent_config.server_cert_file { + if let Some(cert_file_path) = &agent_config.server_cert_file { export_env("KRUSTLET_CERT_FILE", cert_file_path.to_str().unwrap()); } else { warn!("Not exporting server cert file path, as non was specified that could be converted to a String."); } - if let Some(key_file_path) = agent_config.server_key_file { + if let Some(key_file_path) = &agent_config.server_key_file { export_env("KRUSTLET_PRIVATE_KEY_FILE", key_file_path.to_str().unwrap()); } else { warn!("Not exporting server key file path, as non was specified that could be converted to a String."); } info!("args: {:?}", env::args()); - let krustlet_config = Config::new_from_flags(env!("CARGO_PKG_VERSION")); + + let server_config = ServerConfig { + addr: agent_config.server_ip_address.clone(), + port: agent_config.server_port, + cert_file: agent_config.server_cert_file.unwrap_or_default(), + private_key_file: agent_config.server_key_file.unwrap_or_default(), + }; + + let krustlet_config = Config { + node_ip: agent_config.server_ip_address, + hostname: agent_config.hostname.clone(), + node_name: agent_config.hostname, + server_config, + data_dir: agent_config.data_directory, + plugins_dir: Default::default(), + node_labels: agent_config.tags, + // TODO: Discuss whether we want this configurable or leave it at a high number for now + max_pods: 110, + bootstrap_file: agent_config.bootstrap_file, + allow_local_modules: false, + insecure_registries: None, + }; let kubeconfig = KubeConfig::from_kubeconfig(&KubeConfigOptions::default()) .await