From 125a461c34eb3c18f9d34a3bad6655c48da6c14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 10 Feb 2021 14:10:23 +0100 Subject: [PATCH 1/6] Changed the pod cidr that our kubelet reports to the cluster to an empty string, instead of the hard coded default of 10.244.0.0 which causes issues for clusters that have something else configured. This should cause Kubernetes to assign a valid cidr from the configured range. --- src/provider/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/provider/mod.rs b/src/provider/mod.rs index 2071740..76a7ed2 100644 --- a/src/provider/mod.rs +++ b/src/provider/mod.rs @@ -151,6 +151,7 @@ impl Provider for StackableProvider { async fn node(&self, builder: &mut Builder) -> anyhow::Result<()> { builder.set_architecture(Self::ARCH); + builder.set_pod_cidr(""); builder.add_taint("NoSchedule", "kubernetes.io/arch", Self::ARCH); builder.add_taint("NoExecute", "kubernetes.io/arch", Self::ARCH); Ok(()) From 3b1476fb83294b0bfa5eecde1e88ee68a0ec076f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 10 Feb 2021 17:47:11 +0100 Subject: [PATCH 2/6] Added command line parameter to control the setting of pod cidr range. Also changed the default for this value to an empty string. --- documentation/commandline_args.adoc | 102 ++++++++++-------- src/bin/agent.rs | 5 +- src/config/config_documentation/pod_cidr.adoc | 5 + src/config/mod.rs | 20 ++++ src/provider/mod.rs | 8 +- 5 files changed, 94 insertions(+), 46 deletions(-) create mode 100644 src/config/config_documentation/pod_cidr.adoc diff --git a/documentation/commandline_args.adoc b/documentation/commandline_args.adoc index f3f6ec7..7e64ec3 100644 --- a/documentation/commandline_args.adoc +++ b/documentation/commandline_args.adoc @@ -1,123 +1,126 @@ -=== tag +=== server-cert-file -*Default value*: No default value +*Default value*: `No default value` *Required*: false -*Multiple values:* true +*Multiple values:* false -A "key=value" pair that should be assigned to this agent as tag. This can be specified multiple times to assign additional tags. - -Tags are the main way of identifying nodes to assign services to later on. +The certificate file for the local webserver which the Krustlet starts. -=== server-key-file +=== server-port -*Default value*: No default value +*Default value*: `3000` *Required*: false *Multiple values:* false -Private key file (in PKCS8 format) to use for the local webserver the Krustlet starts. +Port to listen on for callbacks. -=== log-directory +=== package-directory -*Default value*: /opt/stackable/logs +*Default value*: `/opt/stackable/packages` *Required*: false *Multiple values:* false -This directory will serve as starting point for all log files which this service creates. -Every service will get its own subdirectory created within this directory. -Anything that is then specified in the log4j config or similar files will be resolved relatively to this directory. +This directory will serve as starting point for packages that are needed by pods assigned to this node.\n Packages will be downloaded into the "_download" folder at the top level of this folder as archives and remain there for potential future use. + +Archives will the be extracted directly into this folder in subdirectories following the naming +scheme of "productname-productversion". The agent will need full access to this directory and tries to create it if it does not exist. -=== hostname +=== data-directory -*Default value*: No default value +*Default value*: `/var/stackable/agent/data` *Required*: false *Multiple values:* false -The hostname to register the node under in Kubernetes - defaults to system hostname. +The directory where the stackable agent should keep its working data. -=== server-cert-file +=== no-config -*Default value*: No default value +*Default value*: `No default value` *Required*: false *Multiple values:* false -The certificate file for the local webserver which the Krustlet starts. +If this option is specified, any file referenced in AGENT_CONF environment variable will be ignored. -=== no-config +=== hostname -*Default value*: No default value +*Default value*: `No default value` *Required*: false *Multiple values:* false -If this option is specified, any file referenced in AGENT_CONF environment variable will be ignored. +The hostname to register the node under in Kubernetes - defaults to system hostname. -=== bootstrap-file +=== pod-cidr -*Default value*: /etc/kubernetes/bootstrap-kubelet.conf +*Default value*: `` *Required*: false *Multiple values:* false -The bootstrap file to use in case Kubernetes bootstraping is used to add the agent. +This setting controls the pod address range that the agent reports to Kubernetes. +The effect of this setting is that Kubernetes will reserve address blocks from withhin this range for every node. +Depending on the setting for maximum pods per node, these will be larger or smaller ranges, and influence the maximum number of nodes for the cluster. +The agent does not require any pod address ranges, and by default doesn't specify anything for this setting. -=== server-bind-ip -*Default value*: No default value +=== server-key-file + +*Default value*: `No default value` *Required*: false *Multiple values:* false -The local IP to register as the node's ip with the apiserver. Will be automatically set to the first address of the first non-loopback interface if not specified. +Private key file (in PKCS8 format) to use for the local webserver the Krustlet starts. -=== data-directory +=== server-bind-ip -*Default value*: /var/stackable/agent/data +*Default value*: `No default value` *Required*: false *Multiple values:* false -The directory where the stackable agent should keep its working data. +The local IP to register as the node's ip with the apiserver. Will be automatically set to the first address of the first non-loopback interface if not specified. === config-directory -*Default value*: /opt/stackable/config +*Default value*: `/opt/stackable/config` *Required*: false @@ -137,30 +140,43 @@ WARNING: This allows anybody who can specify pods more or less full access to th The agent will need full access to this directory and tries to create it if it does not exist. -=== server-port +=== tag -*Default value*: 3000 +*Default value*: `No default value` *Required*: false -*Multiple values:* false +*Multiple values:* true -Port to listen on for callbacks. +A "key=value" pair that should be assigned to this agent as tag. This can be specified multiple times to assign additional tags. + +Tags are the main way of identifying nodes to assign services to later on. -=== package-directory +=== log-directory -*Default value*: /opt/stackable/packages +*Default value*: `/opt/stackable/logs` *Required*: false *Multiple values:* false -This directory will serve as starting point for packages that are needed by pods assigned to this node.\n Packages will be downloaded into the "_download" folder at the top level of this folder as archives and remain there for potential future use. +This directory will serve as starting point for all log files which this service creates. +Every service will get its own subdirectory created within this directory. +Anything that is then specified in the log4j config or similar files will be resolved relatively to this directory. + +The agent will need full access to this directory and tries to create it if it does not exist. + + +=== bootstrap-file + +*Default value*: `/etc/kubernetes/bootstrap-kubelet.conf` + +*Required*: false + +*Multiple values:* false -Archives will the be extracted directly into this folder in subdirectories following the naming -scheme of "productname-productversion". -The agent will need full access to this directory and tries to create it if it does not exist. \ No newline at end of file +The bootstrap file to use in case Kubernetes bootstraping is used to add the agent. \ No newline at end of file diff --git a/src/bin/agent.rs b/src/bin/agent.rs index 74f8641..f53d3db 100644 --- a/src/bin/agent.rs +++ b/src/bin/agent.rs @@ -68,8 +68,8 @@ async fn main() -> anyhow::Result<()> { hostname: agent_config.hostname.clone(), node_name: agent_config.hostname, server_config, - data_dir: agent_config.data_directory, - plugins_dir: Default::default(), + data_dir: agent_config.data_directory.clone(), + plugins_dir: agent_config.data_directory.join("plugins"), node_labels: agent_config.tags, // TODO: Discuss whether we want this configurable or leave it at a high number for now max_pods: 110, @@ -87,6 +87,7 @@ async fn main() -> anyhow::Result<()> { agent_config.parcel_directory.clone(), agent_config.config_directory.clone(), agent_config.log_directory.clone(), + agent_config.pod_cidr, ) .await .expect("Error initializing provider."); diff --git a/src/config/config_documentation/pod_cidr.adoc b/src/config/config_documentation/pod_cidr.adoc new file mode 100644 index 0000000..7eecc16 --- /dev/null +++ b/src/config/config_documentation/pod_cidr.adoc @@ -0,0 +1,5 @@ +This setting controls the pod address range that the agent reports to Kubernetes. +The effect of this setting is that Kubernetes will reserve address blocks from withhin this range for every node. +Depending on the setting for maximum pods per node, these will be larger or smaller ranges, and influence the maximum number of nodes for the cluster. + +The agent does not require any pod address ranges, and by default doesn't specify anything for this setting. \ No newline at end of file diff --git a/src/config/mod.rs b/src/config/mod.rs index ab7923b..c17361b 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -33,6 +33,7 @@ pub struct AgentConfig { pub server_cert_file: Option, pub server_key_file: Option, pub tags: HashMap, + pub pod_cidr: String, } impl AgentConfig { @@ -158,6 +159,16 @@ impl AgentConfig { list: true }; + pub const POD_CIDR: ConfigOption = ConfigOption { + name: "pod-cidr", + default: Some(""), + required: false, + takes_argument: true, + help: "An ip range in CIDR notation which designates the range that pods assigned to this node should have their ip addresses in.", + documentation: include_str!("config_documentation/pod_cidr.adoc"), + list: false + }; + fn get_options() -> HashSet { [ AgentConfig::HOSTNAME, @@ -172,6 +183,7 @@ impl AgentConfig { AgentConfig::NO_CONFIG, AgentConfig::TAG, AgentConfig::BOOTSTRAP_FILE, + AgentConfig::POD_CIDR, ] .iter() .cloned() @@ -376,6 +388,13 @@ impl Configurable for AgentConfig { error_list.as_mut(), ); + // Parse pod cidr + let final_pod_cidr: Result = AgentConfig::get_with_default( + &parsed_values, + &AgentConfig::POD_CIDR, + error_list.as_mut(), + ); + // Parse cert file let final_server_cert_file = if let Ok(server_cert_file) = AgentConfig::get_exactly_one_string(&parsed_values, &AgentConfig::SERVER_CERT_FILE) @@ -454,6 +473,7 @@ impl Configurable for AgentConfig { server_cert_file: final_server_cert_file, server_key_file: final_server_key_file, tags: final_tags, + pod_cidr: final_pod_cidr.unwrap(), }) } } diff --git a/src/provider/mod.rs b/src/provider/mod.rs index 76a7ed2..da32cc2 100644 --- a/src/provider/mod.rs +++ b/src/provider/mod.rs @@ -27,6 +27,7 @@ pub struct StackableProvider { parcel_directory: PathBuf, config_directory: PathBuf, log_directory: PathBuf, + pod_cidr: String, } pub const CRDS: &[&str] = &["repositories.stable.stackable.de"]; @@ -70,12 +71,14 @@ impl StackableProvider { parcel_directory: PathBuf, config_directory: PathBuf, log_directory: PathBuf, + pod_cidr: String, ) -> Result { let provider = StackableProvider { client, parcel_directory, config_directory, log_directory, + pod_cidr, }; let missing_crds = provider.check_crds().await?; return if missing_crds.is_empty() { @@ -150,8 +153,11 @@ impl Provider for StackableProvider { const ARCH: &'static str = "stackable-linux"; async fn node(&self, builder: &mut Builder) -> anyhow::Result<()> { + println!( + "======================================================================\n\n\n\n\n\n" + ); builder.set_architecture(Self::ARCH); - builder.set_pod_cidr(""); + builder.set_pod_cidr(&self.pod_cidr); builder.add_taint("NoSchedule", "kubernetes.io/arch", Self::ARCH); builder.add_taint("NoExecute", "kubernetes.io/arch", Self::ARCH); Ok(()) From bcfcdef766658798d7796f7587f468bdc3c4dc91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 10 Feb 2021 17:49:35 +0100 Subject: [PATCH 3/6] Reverted change that is already included in a PR in review. --- src/bin/agent.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/agent.rs b/src/bin/agent.rs index f53d3db..011a440 100644 --- a/src/bin/agent.rs +++ b/src/bin/agent.rs @@ -68,8 +68,8 @@ async fn main() -> anyhow::Result<()> { hostname: agent_config.hostname.clone(), node_name: agent_config.hostname, server_config, - data_dir: agent_config.data_directory.clone(), - plugins_dir: agent_config.data_directory.join("plugins"), + data_dir: agent_config.data_directory, + plugins_dir: Default::default(), node_labels: agent_config.tags, // TODO: Discuss whether we want this configurable or leave it at a high number for now max_pods: 110, From f508d6a55f5447a54e0bbadb7ef100e07d429af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Wed, 10 Feb 2021 17:50:47 +0100 Subject: [PATCH 4/6] Removed debug log --- src/provider/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/provider/mod.rs b/src/provider/mod.rs index da32cc2..d865778 100644 --- a/src/provider/mod.rs +++ b/src/provider/mod.rs @@ -153,9 +153,6 @@ impl Provider for StackableProvider { const ARCH: &'static str = "stackable-linux"; async fn node(&self, builder: &mut Builder) -> anyhow::Result<()> { - println!( - "======================================================================\n\n\n\n\n\n" - ); builder.set_architecture(Self::ARCH); builder.set_pod_cidr(&self.pod_cidr); builder.add_taint("NoSchedule", "kubernetes.io/arch", Self::ARCH); From c2a0ec59d5062f2640d231202fbd4b3c84f33400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Thu, 11 Feb 2021 06:35:15 +0100 Subject: [PATCH 5/6] Set rust channel to stable for clippy action to see if that fixes the crash --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7ac2168..e168758 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly + toolchain: stable components: clippy, rustfmt override: true - uses: actions-rs/clippy-check@v1 From 63a0adf2fe058054430c8b4b8d617e4071136534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Liebau?= Date: Thu, 11 Feb 2021 06:45:50 +0100 Subject: [PATCH 6/6] Addressed comments from Lars. --- src/config/config_documentation/pod_cidr.adoc | 4 +++- src/config/mod.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/config/config_documentation/pod_cidr.adoc b/src/config/config_documentation/pod_cidr.adoc index 7eecc16..c129031 100644 --- a/src/config/config_documentation/pod_cidr.adoc +++ b/src/config/config_documentation/pod_cidr.adoc @@ -2,4 +2,6 @@ This setting controls the pod address range that the agent reports to Kubernetes The effect of this setting is that Kubernetes will reserve address blocks from withhin this range for every node. Depending on the setting for maximum pods per node, these will be larger or smaller ranges, and influence the maximum number of nodes for the cluster. -The agent does not require any pod address ranges, and by default doesn't specify anything for this setting. \ No newline at end of file +The agent does not require any pod address ranges, and by default doesn't specify anything for this setting. + +WARNING: There should almost never be a reason to use this setting, this is mostly here for very special circumstances. Do not touch it unless you really know what you're doing. \ No newline at end of file diff --git a/src/config/mod.rs b/src/config/mod.rs index c17361b..178e243 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -164,7 +164,7 @@ impl AgentConfig { default: Some(""), required: false, takes_argument: true, - help: "An ip range in CIDR notation which designates the range that pods assigned to this node should have their ip addresses in.", + help: "An IP range in CIDR notation which designates the range that pods assigned to this node should have their ip addresses in.", documentation: include_str!("config_documentation/pod_cidr.adoc"), list: false };