From 23409681c23b78e4b70781292b832792bd14af28 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Fri, 23 Feb 2018 13:44:52 -0500 Subject: [PATCH 1/4] Retrieve cluster location from kube-env, rather than using the instance zone. --- src/configuration.cc | 5 +++++ src/configuration.h | 5 +++++ src/environment.cc | 24 ++++++++++++++++++++++++ src/environment.h | 2 ++ src/kubernetes.cc | 12 ++++++------ 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/configuration.cc b/src/configuration.cc index 53445e32..ad0e5e8c 100644 --- a/src/configuration.cc +++ b/src/configuration.cc @@ -52,6 +52,7 @@ constexpr const char kKubernetesDefaultEndpointHost[] = "https://kubernetes.default.svc"; constexpr const char kKubernetesDefaultPodLabelSelector[] = ""; constexpr const char kKubernetesDefaultClusterName[] = ""; +constexpr const char kKubernetesDefaultClusterLocation[] = ""; constexpr const char kKubernetesDefaultNodeName[] = ""; constexpr const bool kKubernetesDefaultUseWatch = true; constexpr const char kDefaultInstanceId[] = ""; @@ -87,6 +88,7 @@ MetadataAgentConfiguration::MetadataAgentConfiguration() kubernetes_endpoint_host_(kKubernetesDefaultEndpointHost), kubernetes_pod_label_selector_(kKubernetesDefaultPodLabelSelector), kubernetes_cluster_name_(kKubernetesDefaultClusterName), + kubernetes_cluster_location_(kKubernetesDefaultClusterLocation), kubernetes_node_name_(kKubernetesDefaultNodeName), kubernetes_use_watch_(kKubernetesDefaultUseWatch), instance_id_(kDefaultInstanceId), @@ -183,6 +185,9 @@ void MetadataAgentConfiguration::ParseConfigFile(const std::string& filename) { kubernetes_cluster_name_ = config["KubernetesClusterName"].as( kKubernetesDefaultClusterName); + kubernetes_cluster_location_ = + config["KubernetesClusterLocation"].as( + kKubernetesDefaultClusterLocation); kubernetes_node_name_ = config["KubernetesNodeName"].as(kKubernetesDefaultNodeName); kubernetes_use_watch_ = diff --git a/src/configuration.h b/src/configuration.h index d8a9b948..b5a74081 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -112,6 +112,10 @@ class MetadataAgentConfiguration { std::lock_guard lock(mutex_); return kubernetes_cluster_name_; } + const std::string& KubernetesClusterLocation() const { + std::lock_guard lock(mutex_); + return kubernetes_cluster_location_; + } const std::string& KubernetesNodeName() const { std::lock_guard lock(mutex_); return kubernetes_node_name_; @@ -154,6 +158,7 @@ class MetadataAgentConfiguration { std::string kubernetes_endpoint_host_; std::string kubernetes_pod_label_selector_; std::string kubernetes_cluster_name_; + std::string kubernetes_cluster_location_; std::string kubernetes_node_name_; bool kubernetes_use_watch_; std::string instance_id_; diff --git a/src/environment.cc b/src/environment.cc index 6f3969e4..26aa804b 100644 --- a/src/environment.cc +++ b/src/environment.cc @@ -18,6 +18,7 @@ #include #include +#include #include "format.h" #include "logging.h" @@ -205,6 +206,29 @@ const std::string& Environment::KubernetesClusterName() const { return kubernetes_cluster_name_; } +const std::string& Environment::KubernetesClusterLocation() const { + std::lock_guard lock(mutex_); + if (kubernetes_cluster_location_.empty()) { + if (!config_.KubernetesClusterLocation().empty()) { + kubernetes_cluster_location_ = config_.KubernetesClusterLocation(); + } else { + // Get the kube-env. + const std::string kube_env = + GetMetadataString("instance/attributes/kube-env"); + // kube-env is a list of NAME: VALUE pairs, one per line. + // The actual location is in the ZONE variable. + // TODO: Refactor this into GetKubeEnv("ZONE") in kubernetes.cc. + std::istringstream in(kube_env); + for (std::string line; std::getline(in, line); ) { + if (line.find("ZONE: ") == 0) { + kubernetes_cluster_location_ = line.substr(line.find(':') + 2); + } + } + } + } + return kubernetes_cluster_location_; +} + const std::string& Environment::CredentialsClientEmail() const { std::lock_guard lock(mutex_); ReadApplicationDefaultCredentials(); diff --git a/src/environment.h b/src/environment.h index 19754f43..5f79491c 100644 --- a/src/environment.h +++ b/src/environment.h @@ -33,6 +33,7 @@ class Environment { const std::string& InstanceId() const; const std::string& InstanceZone() const; const std::string& KubernetesClusterName() const; + const std::string& KubernetesClusterLocation() const; const std::string& CredentialsClientEmail() const; const std::string& CredentialsPrivateKey() const; @@ -54,6 +55,7 @@ class Environment { mutable std::string instance_id_; mutable std::string instance_resource_type_; mutable std::string kubernetes_cluster_name_; + mutable std::string kubernetes_cluster_location_; mutable std::string client_email_; mutable std::string private_key_; mutable bool application_default_credentials_read_; diff --git a/src/kubernetes.cc b/src/kubernetes.cc index c4be0341..788f2b22 100644 --- a/src/kubernetes.cc +++ b/src/kubernetes.cc @@ -86,8 +86,8 @@ KubernetesReader::KubernetesReader(const MetadataAgentConfiguration& config) MetadataUpdater::ResourceMetadata KubernetesReader::GetNodeMetadata( json::value raw_node, Timestamp collected_at) const throw(json::Exception) { - const std::string zone = environment_.InstanceZone(); const std::string cluster_name = environment_.KubernetesClusterName(); + const std::string location = environment_.KubernetesClusterLocation(); const json::Object* node = raw_node->As(); const json::Object* metadata = node->Get("metadata"); @@ -99,7 +99,7 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetNodeMetadata( const MonitoredResource k8s_node("k8s_node", { {"cluster_name", cluster_name}, {"node_name", node_name}, - {"location", zone}, + {"location", location}, }); json::value instance_resource = @@ -185,8 +185,8 @@ json::value KubernetesReader::ComputePodAssociations(const json::Object* pod) MetadataUpdater::ResourceMetadata KubernetesReader::GetPodMetadata( json::value raw_pod, json::value associations, Timestamp collected_at) const throw(json::Exception) { - const std::string zone = environment_.InstanceZone(); const std::string cluster_name = environment_.KubernetesClusterName(); + const std::string location = environment_.KubernetesClusterLocation(); const json::Object* pod = raw_pod->As(); @@ -206,7 +206,7 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetPodMetadata( {"cluster_name", cluster_name}, {"namespace_name", namespace_name}, {"pod_name", pod_name}, - {"location", zone}, + {"location", location}, }); // TODO: find is_deleted. @@ -249,8 +249,8 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetContainerMetadata( const json::Object* pod, const json::Object* container_spec, const json::Object* container_status, json::value associations, Timestamp collected_at) const throw(json::Exception) { - const std::string zone = environment_.InstanceZone(); const std::string cluster_name = environment_.KubernetesClusterName(); + const std::string location = environment_.KubernetesClusterLocation(); const json::Object* metadata = pod->Get("metadata"); const std::string namespace_name = metadata->Get("namespace"); @@ -276,7 +276,7 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetContainerMetadata( {"namespace_name", namespace_name}, {"pod_name", pod_name}, {"container_name", container_name}, - {"location", zone}, + {"location", location}, }); std::unique_ptr blobs(new json::Object({ From 752ab62e70d95ea7d473e655584962074f347543 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Fri, 23 Feb 2018 19:50:18 -0500 Subject: [PATCH 2/4] Use the cluster-location instance attribute instead. --- src/environment.cc | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/environment.cc b/src/environment.cc index 26aa804b..08728399 100644 --- a/src/environment.cc +++ b/src/environment.cc @@ -212,18 +212,10 @@ const std::string& Environment::KubernetesClusterLocation() const { if (!config_.KubernetesClusterLocation().empty()) { kubernetes_cluster_location_ = config_.KubernetesClusterLocation(); } else { - // Get the kube-env. - const std::string kube_env = - GetMetadataString("instance/attributes/kube-env"); - // kube-env is a list of NAME: VALUE pairs, one per line. - // The actual location is in the ZONE variable. - // TODO: Refactor this into GetKubeEnv("ZONE") in kubernetes.cc. - std::istringstream in(kube_env); - for (std::string line; std::getline(in, line); ) { - if (line.find("ZONE: ") == 0) { - kubernetes_cluster_location_ = line.substr(line.find(':') + 2); - } - } + // Query the metadata server. + // TODO: Other sources? kube-env? + kubernetes_cluster_location_ = + GetMetadataString("instance/attributes/cluster-location"); } } return kubernetes_cluster_location_; From 8bf02385a385f79587d4062c2c35460031fc30be Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Sat, 24 Feb 2018 13:13:13 -0500 Subject: [PATCH 3/4] Remove unneeded header. --- src/environment.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/environment.cc b/src/environment.cc index 08728399..23f950c4 100644 --- a/src/environment.cc +++ b/src/environment.cc @@ -18,7 +18,6 @@ #include #include -#include #include "format.h" #include "logging.h" From f30ec5c22ad378242c503103c7b000bad8858334 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Sun, 25 Feb 2018 00:06:41 -0500 Subject: [PATCH 4/4] Fall back on kube-env if the newer attribute is not available. --- src/environment.cc | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/environment.cc b/src/environment.cc index 23f950c4..349b3057 100644 --- a/src/environment.cc +++ b/src/environment.cc @@ -18,6 +18,7 @@ #include #include +#include #include "format.h" #include "logging.h" @@ -212,9 +213,24 @@ const std::string& Environment::KubernetesClusterLocation() const { kubernetes_cluster_location_ = config_.KubernetesClusterLocation(); } else { // Query the metadata server. - // TODO: Other sources? kube-env? kubernetes_cluster_location_ = GetMetadataString("instance/attributes/cluster-location"); + // Fall back on kube-env for older clusters. + if (kubernetes_cluster_location_.empty()) { + // Get the kube-env. + const std::string kube_env = + GetMetadataString("instance/attributes/kube-env"); + // kube-env is a list of NAME: VALUE pairs, one per line, unsorted. + // The actual location is in the ZONE variable. + // TODO: Refactor this into GetKubeEnv("ZONE") in kubernetes.cc. + std::istringstream in(kube_env); + for (std::string line; std::getline(in, line); ) { + if (line.find("ZONE: ") == 0) { + kubernetes_cluster_location_ = line.substr(line.find(':') + 2); + break; + } + } + } } } return kubernetes_cluster_location_;