From 69068045162d100de184482e68a05b2761835f7c Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Thu, 22 Feb 2018 13:32:09 -0500 Subject: [PATCH 1/5] Turn MetadataApiResourceTypeSeparator and MetadataIngestionRawContentVersion into configuration options. --- src/configuration.cc | 15 ++++++++++++++- src/configuration.h | 10 ++++++++++ src/docker.cc | 22 +++++++++++----------- src/kubernetes.cc | 27 ++++++++++++--------------- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/src/configuration.cc b/src/configuration.cc index 105cc2a1..bd4264a1 100644 --- a/src/configuration.cc +++ b/src/configuration.cc @@ -31,19 +31,22 @@ constexpr const char kDefaultProjectId[] = ""; constexpr const char kDefaultCredentialsFile[] = ""; constexpr const int kMetadataApiDefaultNumThreads = 3; constexpr const int kMetadataApiDefaultPort = 8000; +constexpr const char kMetadataApiDefaultResourceTypeSeparator[] = "."; constexpr const int kMetadataReporterDefaultIntervalSeconds = 60; constexpr const char kMetadataIngestionDefaultEndpointFormat[] = "https://stackdriver.googleapis.com/v1beta2/projects/{{project_id}}" "/resourceMetadata:batchUpdate"; constexpr const int kMetadataIngestionDefaultRequestSizeLimitBytes = 8*1024*1024; +constexpr const char kMetadataIngestionDefaultRawContentVersion[] = "0.1"; constexpr const int kDockerUpdaterDefaultIntervalSeconds = 60; constexpr const char kDockerDefaultEndpointHost[] = "unix://%2Fvar%2Frun%2Fdocker.sock/"; constexpr const char kDockerDefaultApiVersion[] = "1.23"; constexpr const char kDockerDefaultContainerFilter[] = "limit=30"; constexpr const int kKubernetesUpdaterDefaultIntervalSeconds = 60; -constexpr const char kKubernetesDefaultEndpointHost[] = "https://kubernetes.default.svc"; +constexpr const char kKubernetesDefaultEndpointHost[] = + "https://kubernetes.default.svc"; constexpr const char kKubernetesDefaultPodLabelSelector[] = ""; constexpr const char kKubernetesDefaultClusterName[] = ""; constexpr const char kKubernetesDefaultNodeName[] = ""; @@ -59,12 +62,16 @@ MetadataAgentConfiguration::MetadataAgentConfiguration() verbose_logging_(false), metadata_api_num_threads_(kMetadataApiDefaultNumThreads), metadata_api_port_(kMetadataApiDefaultPort), + metadata_api_resource_type_separator_( + kMetadataApiDefaultResourceTypeSeparator), metadata_reporter_interval_seconds_( kMetadataReporterDefaultIntervalSeconds), metadata_ingestion_endpoint_format_( kMetadataIngestionDefaultEndpointFormat), metadata_ingestion_request_size_limit_bytes_( kMetadataIngestionDefaultRequestSizeLimitBytes), + metadata_ingestion_raw_content_version_( + kMetadataIngestionDefaultRawContentVersion), docker_updater_interval_seconds_(kDockerUpdaterDefaultIntervalSeconds), docker_endpoint_host_(kDockerDefaultEndpointHost), docker_api_version_(kDockerDefaultApiVersion), @@ -126,6 +133,9 @@ void MetadataAgentConfiguration::ParseConfigFile(const std::string& filename) { config["MetadataApiNumThreads"].as(kMetadataApiDefaultNumThreads); metadata_api_port_ = config["MetadataApiPort"].as(kMetadataApiDefaultPort); + metadata_api_resource_type_separator_ = + config["MetadataApiResourceTypeSeparator"].as( + kMetadataApiDefaultResourceTypeSeparator); metadata_reporter_interval_seconds_ = config["MetadataReporterIntervalSeconds"].as( kMetadataReporterDefaultIntervalSeconds); @@ -135,6 +145,9 @@ void MetadataAgentConfiguration::ParseConfigFile(const std::string& filename) { metadata_ingestion_request_size_limit_bytes_ = config["MetadataIngestionRequestSizeLimitBytes"].as( kMetadataIngestionDefaultRequestSizeLimitBytes); + metadata_ingestion_raw_content_version_ = + config["MetadataIngestionRawContentVersion"].as( + kMetadataIngestionDefaultRawContentVersion); docker_updater_interval_seconds_ = config["DockerUpdaterIntervalSeconds"].as( kDockerUpdaterDefaultIntervalSeconds); diff --git a/src/configuration.h b/src/configuration.h index c38da470..fb677178 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -48,6 +48,10 @@ class MetadataAgentConfiguration { std::lock_guard lock(mutex_); return metadata_api_port_; } + const std::string& MetadataApiResourceTypeSeparator() const { + std::lock_guard lock(mutex_); + return metadata_api_resource_type_separator_; + } // Metadata reporter options. int MetadataReporterIntervalSeconds() const { std::lock_guard lock(mutex_); @@ -61,6 +65,10 @@ class MetadataAgentConfiguration { std::lock_guard lock(mutex_); return metadata_ingestion_request_size_limit_bytes_; } + const std::string& MetadataIngestionRawContentVersion() const { + std::lock_guard lock(mutex_); + return metadata_ingestion_raw_content_version_; + } // Docker metadata updater options. int DockerUpdaterIntervalSeconds() const { std::lock_guard lock(mutex_); @@ -122,9 +130,11 @@ class MetadataAgentConfiguration { bool verbose_logging_; int metadata_api_num_threads_; int metadata_api_port_; + std::string metadata_api_resource_type_separator_; int metadata_reporter_interval_seconds_; std::string metadata_ingestion_endpoint_format_; int metadata_ingestion_request_size_limit_bytes_; + std::string metadata_ingestion_raw_content_version_; int docker_updater_interval_seconds_; std::string docker_endpoint_host_; std::string docker_api_version_; diff --git a/src/docker.cc b/src/docker.cc index f90454db..aff50e4a 100644 --- a/src/docker.cc +++ b/src/docker.cc @@ -32,11 +32,10 @@ namespace google { namespace { #if 0 -constexpr const char docker_endpoint_host[] = "unix://%2Fvar%2Frun%2Fdocker.sock/"; -constexpr const char docker_api_version[] = "1.23"; +constexpr const char kDockerEndpointHost[] = "unix://%2Fvar%2Frun%2Fdocker.sock/"; +constexpr const char kDockerApiVersion[] = "1.23"; #endif -constexpr const char docker_endpoint_path[] = "/containers"; -constexpr const char resource_type_separator[] = "."; +constexpr const char kDockerEndpointPath[] = "/containers"; } @@ -51,7 +50,7 @@ std::vector const std::string zone = environment_.InstanceZone(); const std::string docker_endpoint(config_.DockerEndpointHost() + "v" + config_.DockerApiVersion() + - docker_endpoint_path); + kDockerEndpointPath); const std::string container_filter( config_.DockerContainerFilter().empty() ? "" : "&" + config_.DockerContainerFilter()); @@ -80,17 +79,18 @@ std::vector if (config_.VerboseLogging()) { LOG(INFO) << "Inspect response: " << body(inspect_response); } - json::value parsed_metadata = + json::value raw_docker = json::Parser::FromString(body(inspect_response)); if (config_.VerboseLogging()) { - LOG(INFO) << "Parsed metadata: " << *parsed_metadata; + LOG(INFO) << "Parsed metadata: " << *raw_docker; } + const MonitoredResource resource("docker_container", { {"location", zone}, {"container_id", id}, }); - const json::Object* container_desc = parsed_metadata->As(); + const json::Object* container_desc = raw_docker->As(); const std::string name = container_desc->Get("Name"); const std::string created_str = @@ -101,15 +101,15 @@ std::vector bool is_deleted = state->Get("Dead"); const std::string resource_id = - std::string("container") + resource_type_separator + id; + std::string("container") + config_.MetadataApiResourceTypeSeparator() + id; // The container name reported by Docker will always have a leading '/'. const std::string resource_name = - std::string("container") + resource_type_separator + name.substr(1); + std::string("container") + config_.MetadataApiResourceTypeSeparator() + name.substr(1); result.emplace_back(std::vector{resource_id, resource_name}, resource, MetadataAgent::Metadata(config_.DockerApiVersion(), is_deleted, created_at, collected_at, - std::move(parsed_metadata))); + std::move(raw_docker))); } catch (const json::Exception& e) { LOG(ERROR) << e.what(); continue; diff --git a/src/kubernetes.cc b/src/kubernetes.cc index fa70c121..463b89a2 100644 --- a/src/kubernetes.cc +++ b/src/kubernetes.cc @@ -43,9 +43,6 @@ constexpr const char kKubernetesEndpointHost[] = "https://kubernetes.default.svc #endif constexpr const char kKubernetesApiVersion[] = "1.6"; constexpr const char kKubernetesEndpointPath[] = "/api/v1"; -constexpr const char kResourceTypeSeparator[] = "."; - -constexpr const char kRawContentVersion[] = "0.1"; constexpr const char kGkeContainerResourcePrefix[] = "gke_container"; constexpr const char kK8sContainerResourcePrefix[] = "k8s_container"; @@ -119,7 +116,7 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetNodeMetadata( json::value node_raw_metadata = json::object({ {"blobs", json::object({ {"association", json::object({ - {"version", json::string(kRawContentVersion)}, + {"version", json::string(config_.MetadataIngestionRawContentVersion())}, {"raw", json::object({ {"infrastructureResource", std::move(InstanceResource())}, })}, @@ -136,12 +133,12 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetNodeMetadata( const std::string k8s_node_name = boost::algorithm::join( std::vector{kK8sNodeResourcePrefix, node_name}, - kResourceTypeSeparator); + config_.MetadataApiResourceTypeSeparator()); return MetadataUpdater::ResourceMetadata( std::vector{k8s_node_name}, k8s_node, #ifdef ENABLE_KUBERNETES_METADATA - MetadataAgent::Metadata(kRawContentVersion, + MetadataAgent::Metadata(config_.MetadataIngestionRawContentVersion(), /*deleted=*/false, created_at, collected_at, std::move(node_raw_metadata)) #else @@ -178,7 +175,7 @@ json::value KubernetesReader::ComputePodAssociations(const json::Object* pod) const std::string node_name = spec->Get("nodeName"); return json::object({ - {"version", json::string(kRawContentVersion)}, + {"version", json::string(config_.MetadataIngestionRawContentVersion())}, {"raw", json::object({ {"infrastructureResource", std::move(InstanceResource())}, {"controllers", json::object({ @@ -236,15 +233,15 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetPodMetadata( const std::string k8s_pod_id = boost::algorithm::join( std::vector{kK8sPodResourcePrefix, pod_id}, - kResourceTypeSeparator); + config_.MetadataApiResourceTypeSeparator()); const std::string k8s_pod_name = boost::algorithm::join( std::vector{kK8sPodResourcePrefix, namespace_name, pod_name}, - kResourceTypeSeparator); + config_.MetadataApiResourceTypeSeparator()); return MetadataUpdater::ResourceMetadata( std::vector{k8s_pod_id, k8s_pod_name}, k8s_pod, #ifdef ENABLE_KUBERNETES_METADATA - MetadataAgent::Metadata(kRawContentVersion, + MetadataAgent::Metadata(config_.MetadataIngestionRawContentVersion(), is_deleted, created_at, collected_at, std::move(pod_raw_metadata)) #else @@ -321,10 +318,10 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetContainerMetadata( const std::string k8s_container_pod = boost::algorithm::join( std::vector{kK8sContainerResourcePrefix, pod_id, container_name}, - kResourceTypeSeparator); + config_.MetadataApiResourceTypeSeparator()); const std::string k8s_container_name = boost::algorithm::join( std::vector{kK8sContainerResourcePrefix, namespace_name, pod_name, container_name}, - kResourceTypeSeparator); + config_.MetadataApiResourceTypeSeparator()); std::vector local_resource_ids = { k8s_container_pod, @@ -346,7 +343,7 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetContainerMetadata( const std::string container_id = docker_id.substr(docker_prefix_length); const std::string k8s_container_id = boost::algorithm::join( std::vector{kK8sContainerResourcePrefix, container_id}, - kResourceTypeSeparator); + config_.MetadataApiResourceTypeSeparator()); local_resource_ids.push_back(k8s_container_id); } @@ -387,10 +384,10 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetLegacyResource( const std::string gke_container_pod_id = boost::algorithm::join( std::vector{kGkeContainerResourcePrefix, namespace_name, pod_id, container_name}, - kResourceTypeSeparator); + config_.MetadataApiResourceTypeSeparator()); const std::string gke_container_name = boost::algorithm::join( std::vector{kGkeContainerResourcePrefix, namespace_name, pod_name, container_name}, - kResourceTypeSeparator); + config_.MetadataApiResourceTypeSeparator()); return MetadataUpdater::ResourceMetadata( std::vector{gke_container_pod_id, gke_container_name}, gke_container, From cedbf03616748893c235c73e4ae0d9b5f9f73322 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Wed, 21 Feb 2018 15:47:40 -0500 Subject: [PATCH 2/5] Factor out instance resources into an instance updater. --- src/Makefile | 1 + src/configuration.cc | 11 ++++++++ src/configuration.h | 11 ++++++++ src/environment.cc | 18 +++++++++++++ src/environment.h | 2 ++ src/instance.cc | 63 ++++++++++++++++++++++++++++++++++++++++++++ src/instance.h | 56 +++++++++++++++++++++++++++++++++++++++ src/kubernetes.cc | 24 +++++++---------- src/kubernetes.h | 3 --- src/metadatad.cc | 3 +++ 10 files changed, 174 insertions(+), 18 deletions(-) create mode 100644 src/instance.cc create mode 100644 src/instance.h diff --git a/src/Makefile b/src/Makefile index 9b7a4c60..0fec29a0 100644 --- a/src/Makefile +++ b/src/Makefile @@ -27,6 +27,7 @@ SRCS=\ api_server.cc \ configuration.cc \ updater.cc \ + instance.cc \ docker.cc \ kubernetes.cc \ resource.cc \ diff --git a/src/configuration.cc b/src/configuration.cc index bd4264a1..cc1b4d3e 100644 --- a/src/configuration.cc +++ b/src/configuration.cc @@ -39,6 +39,8 @@ constexpr const char kMetadataIngestionDefaultEndpointFormat[] = constexpr const int kMetadataIngestionDefaultRequestSizeLimitBytes = 8*1024*1024; constexpr const char kMetadataIngestionDefaultRawContentVersion[] = "0.1"; +constexpr const int kInstanceUpdaterDefaultIntervalSeconds = 60*60; +constexpr const char kDefaultInstanceResourceType[] = ""; constexpr const int kDockerUpdaterDefaultIntervalSeconds = 60; constexpr const char kDockerDefaultEndpointHost[] = "unix://%2Fvar%2Frun%2Fdocker.sock/"; @@ -72,6 +74,9 @@ MetadataAgentConfiguration::MetadataAgentConfiguration() kMetadataIngestionDefaultRequestSizeLimitBytes), metadata_ingestion_raw_content_version_( kMetadataIngestionDefaultRawContentVersion), + instance_updater_interval_seconds_( + kInstanceUpdaterDefaultIntervalSeconds), + instance_resource_type_(kDefaultInstanceResourceType), docker_updater_interval_seconds_(kDockerUpdaterDefaultIntervalSeconds), docker_endpoint_host_(kDockerDefaultEndpointHost), docker_api_version_(kDockerDefaultApiVersion), @@ -148,6 +153,12 @@ void MetadataAgentConfiguration::ParseConfigFile(const std::string& filename) { metadata_ingestion_raw_content_version_ = config["MetadataIngestionRawContentVersion"].as( kMetadataIngestionDefaultRawContentVersion); + instance_updater_interval_seconds_ = + config["InstanceUpdaterIntervalSeconds"].as( + kInstanceUpdaterDefaultIntervalSeconds); + instance_resource_type_ = + config["InstanceResourceType"].as( + kDefaultInstanceResourceType); docker_updater_interval_seconds_ = config["DockerUpdaterIntervalSeconds"].as( kDockerUpdaterDefaultIntervalSeconds); diff --git a/src/configuration.h b/src/configuration.h index fb677178..d8a9b948 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -69,6 +69,15 @@ class MetadataAgentConfiguration { std::lock_guard lock(mutex_); return metadata_ingestion_raw_content_version_; } + // Instance metadata updater options. + int InstanceUpdaterIntervalSeconds() const { + std::lock_guard lock(mutex_); + return instance_updater_interval_seconds_; + } + const std::string& InstanceResourceType() const { + std::lock_guard lock(mutex_); + return instance_resource_type_; + } // Docker metadata updater options. int DockerUpdaterIntervalSeconds() const { std::lock_guard lock(mutex_); @@ -135,6 +144,8 @@ class MetadataAgentConfiguration { std::string metadata_ingestion_endpoint_format_; int metadata_ingestion_request_size_limit_bytes_; std::string metadata_ingestion_raw_content_version_; + int instance_updater_interval_seconds_; + std::string instance_resource_type_; int docker_updater_interval_seconds_; std::string docker_endpoint_host_; std::string docker_api_version_; diff --git a/src/environment.cc b/src/environment.cc index d06d557e..3ac3eaa0 100644 --- a/src/environment.cc +++ b/src/environment.cc @@ -70,6 +70,10 @@ json::value ReadCredentials( constexpr const char kGceMetadataServerAddress[] = "http://metadata.google.internal./computeMetadata/v1/"; + +constexpr const char kGceInstanceResourceType[] = + "gce_instance"; + } Environment::Environment(const MetadataAgentConfiguration& config) @@ -164,6 +168,20 @@ const std::string& Environment::InstanceId() const { return instance_id_; } +const std::string& Environment::InstanceResourceType() const { + std::lock_guard lock(mutex_); + if (instance_resource_type_.empty()) { + if (!config_.InstanceResourceType().empty()) { + instance_resource_type_ = config_.InstanceResourceType(); + } else { + // Default to a GCE instance. + // TODO: Detect other resources. + instance_resource_type_ = kGceInstanceResourceType; + } + } + return instance_resource_type_; +} + const std::string& Environment::KubernetesClusterName() const { std::lock_guard lock(mutex_); if (kubernetes_cluster_name_.empty()) { diff --git a/src/environment.h b/src/environment.h index 1eaacef2..19754f43 100644 --- a/src/environment.h +++ b/src/environment.h @@ -29,6 +29,7 @@ class Environment { Environment(const MetadataAgentConfiguration& config); const std::string& NumericProjectId() const; + const std::string& InstanceResourceType() const; const std::string& InstanceId() const; const std::string& InstanceZone() const; const std::string& KubernetesClusterName() const; @@ -51,6 +52,7 @@ class Environment { mutable std::string project_id_; mutable std::string zone_; mutable std::string instance_id_; + mutable std::string instance_resource_type_; mutable std::string kubernetes_cluster_name_; mutable std::string client_email_; mutable std::string private_key_; diff --git a/src/instance.cc b/src/instance.cc new file mode 100644 index 00000000..59381176 --- /dev/null +++ b/src/instance.cc @@ -0,0 +1,63 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#include "instance.h" + +#include +#include + +#include "json.h" +#include "logging.h" +#include "time.h" + +namespace http = boost::network::http; + +namespace google { + +InstanceReader::InstanceReader(const MetadataAgentConfiguration& config) + : config_(config), environment_(config) {} + +/*static*/ +MonitoredResource InstanceReader::InstanceResource(const Environment& environment) { + const std::string resource_type = environment.InstanceResourceType(); + const std::string instance_id = environment.InstanceId(); + const std::string zone = environment.InstanceZone(); + return {resource_type, { + {"instance_id", instance_id}, + {"zone", zone}, + }}; +} + +std::vector + InstanceReader::MetadataQuery() const { + if (config_.VerboseLogging()) { + LOG(INFO) << "Instance Query called"; + } + + MonitoredResource instance_resource = InstanceResource(environment_); + + std::vector result; + result.emplace_back( + std::vector{"", environment_.InstanceId()}, + instance_resource, + // TODO: Send actual instance metadata. + MetadataAgent::Metadata::IGNORED() + ); + return result; +} + +} + diff --git a/src/instance.h b/src/instance.h new file mode 100644 index 00000000..db9c4da1 --- /dev/null +++ b/src/instance.h @@ -0,0 +1,56 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ +#ifndef INSTANCE_H_ +#define INSTANCE_H_ + +//#include "config.h" + +#include + +#include "configuration.h" +#include "environment.h" +#include "resource.h" +#include "updater.h" + +namespace google { + +class InstanceReader { + public: + InstanceReader(const MetadataAgentConfiguration& config); + // A Instance metadata query function. + std::vector MetadataQuery() const; + + // Gets the monitored resource of the instance the agent is running on. + static MonitoredResource InstanceResource(const Environment& environment); + + private: + const MetadataAgentConfiguration& config_; + Environment environment_; +}; + +class InstanceUpdater : public PollingMetadataUpdater { + public: + InstanceUpdater(MetadataAgent* server) + : reader_(server->config()), PollingMetadataUpdater( + server, server->config().InstanceUpdaterIntervalSeconds(), + std::bind(&google::InstanceReader::MetadataQuery, &reader_)) { } + private: + InstanceReader reader_; +}; + +} + +#endif // INSTANCE_H_ diff --git a/src/kubernetes.cc b/src/kubernetes.cc index 463b89a2..a79ef2c4 100644 --- a/src/kubernetes.cc +++ b/src/kubernetes.cc @@ -27,6 +27,7 @@ #include #include "http_common.h" +#include "instance.h" #include "json.h" #include "logging.h" #include "resource.h" @@ -82,19 +83,6 @@ bool ReadServiceAccountSecret( KubernetesReader::KubernetesReader(const MetadataAgentConfiguration& config) : config_(config), environment_(config) {} -json::value KubernetesReader::InstanceResource() const { - const std::string resource_type = "gce_instance"; // TODO: detect other resources. - const std::string instance_id = environment_.InstanceId(); - const std::string zone = environment_.InstanceZone(); - return json::object({ - {"type", json::string(resource_type)}, - {"labels", json::object({ - {"instance_id", json::string(instance_id)}, - {"zone", json::string(zone)}, - })}, - }); -} - MetadataUpdater::ResourceMetadata KubernetesReader::GetNodeMetadata( json::value raw_node, Timestamp collected_at) const throw(json::Exception) { const std::string zone = environment_.InstanceZone(); @@ -113,12 +101,15 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetNodeMetadata( {"location", zone}, }); + json::value instance_resource = + InstanceReader::InstanceResource(environment_).ToJSON(); + json::value node_raw_metadata = json::object({ {"blobs", json::object({ {"association", json::object({ {"version", json::string(config_.MetadataIngestionRawContentVersion())}, {"raw", json::object({ - {"infrastructureResource", std::move(InstanceResource())}, + {"infrastructureResource", std::move(instance_resource)}, })}, })}, {"api", json::object({ @@ -174,10 +165,13 @@ json::value KubernetesReader::ComputePodAssociations(const json::Object* pod) const json::Object* spec = pod->Get("spec"); const std::string node_name = spec->Get("nodeName"); + json::value instance_resource = + InstanceReader::InstanceResource(environment_).ToJSON(); + return json::object({ {"version", json::string(config_.MetadataIngestionRawContentVersion())}, {"raw", json::object({ - {"infrastructureResource", std::move(InstanceResource())}, + {"infrastructureResource", std::move(instance_resource)}, {"controllers", json::object({ {"topLevelControllerType", json::string(top_level_kind)}, {"topLevelControllerName", json::string(top_level_name)}, diff --git a/src/kubernetes.h b/src/kubernetes.h index b01ac965..601c6e7d 100644 --- a/src/kubernetes.h +++ b/src/kubernetes.h @@ -127,9 +127,6 @@ class KubernetesReader { json::value FindTopLevelOwner(const std::string& ns, json::value object) const throw(QueryException, json::Exception); - // Gets the monitored resource of the instance the agent is running on. - json::value InstanceResource() const; - // Cached data. mutable std::recursive_mutex mutex_; mutable std::string current_node_; diff --git a/src/metadatad.cc b/src/metadatad.cc index a2645b0a..57d78ac6 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -21,6 +21,7 @@ #include "api_server.h" #include "configuration.h" #include "docker.h" +#include "instance.h" #include "kubernetes.h" int main(int ac, char** av) { @@ -32,9 +33,11 @@ int main(int ac, char** av) { google::MetadataAgent server(config); + google::InstanceUpdater instance_updater(&server); google::DockerUpdater docker_updater(&server); google::KubernetesUpdater kubernetes_updater(&server); + instance_updater.start(); docker_updater.start(); kubernetes_updater.start(); From 83ffd886691524579aece7a8dce0abcf4d389092 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Thu, 22 Feb 2018 14:17:12 -0500 Subject: [PATCH 3/5] Revert docker.cc cleanup. --- src/docker.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/docker.cc b/src/docker.cc index aff50e4a..18361026 100644 --- a/src/docker.cc +++ b/src/docker.cc @@ -32,10 +32,10 @@ namespace google { namespace { #if 0 -constexpr const char kDockerEndpointHost[] = "unix://%2Fvar%2Frun%2Fdocker.sock/"; -constexpr const char kDockerApiVersion[] = "1.23"; +constexpr const char docker_endpoint_host[] = "unix://%2Fvar%2Frun%2Fdocker.sock/"; +constexpr const char docker_api_version[] = "1.23"; #endif -constexpr const char kDockerEndpointPath[] = "/containers"; +constexpr const char docker_endpoint_path[] = "/containers"; } @@ -50,7 +50,7 @@ std::vector const std::string zone = environment_.InstanceZone(); const std::string docker_endpoint(config_.DockerEndpointHost() + "v" + config_.DockerApiVersion() + - kDockerEndpointPath); + docker_endpoint_path); const std::string container_filter( config_.DockerContainerFilter().empty() ? "" : "&" + config_.DockerContainerFilter()); @@ -79,10 +79,10 @@ std::vector if (config_.VerboseLogging()) { LOG(INFO) << "Inspect response: " << body(inspect_response); } - json::value raw_docker = + json::value parsed_metadata = json::Parser::FromString(body(inspect_response)); if (config_.VerboseLogging()) { - LOG(INFO) << "Parsed metadata: " << *raw_docker; + LOG(INFO) << "Parsed metadata: " << *parsed_metadata; } const MonitoredResource resource("docker_container", { @@ -90,7 +90,7 @@ std::vector {"container_id", id}, }); - const json::Object* container_desc = raw_docker->As(); + const json::Object* container_desc = parsed_metadata->As(); const std::string name = container_desc->Get("Name"); const std::string created_str = @@ -109,7 +109,7 @@ std::vector resource, MetadataAgent::Metadata(config_.DockerApiVersion(), is_deleted, created_at, collected_at, - std::move(raw_docker))); + std::move(parsed_metadata))); } catch (const json::Exception& e) { LOG(ERROR) << e.what(); continue; From e39adc62bf9ad1fc76a15c4ea03b6d78e5124599 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Thu, 22 Feb 2018 14:58:23 -0500 Subject: [PATCH 4/5] What year is it?! And other minor tweaks. --- src/environment.cc | 5 ++--- src/instance.cc | 2 +- src/instance.h | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/environment.cc b/src/environment.cc index 3ac3eaa0..87d0e346 100644 --- a/src/environment.cc +++ b/src/environment.cc @@ -71,8 +71,7 @@ json::value ReadCredentials( constexpr const char kGceMetadataServerAddress[] = "http://metadata.google.internal./computeMetadata/v1/"; -constexpr const char kGceInstanceResourceType[] = - "gce_instance"; +constexpr const char kGceInstanceResourceType[] = "gce_instance"; } @@ -175,7 +174,7 @@ const std::string& Environment::InstanceResourceType() const { instance_resource_type_ = config_.InstanceResourceType(); } else { // Default to a GCE instance. - // TODO: Detect other resources. + // TODO: Detect other instance resources. instance_resource_type_ = kGceInstanceResourceType; } } diff --git a/src/instance.cc b/src/instance.cc index 59381176..31a639e8 100644 --- a/src/instance.cc +++ b/src/instance.cc @@ -1,5 +1,5 @@ /* - * Copyright 2017 Google Inc. + * Copyright 2018 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/src/instance.h b/src/instance.h index db9c4da1..6fa612f7 100644 --- a/src/instance.h +++ b/src/instance.h @@ -1,5 +1,5 @@ /* - * Copyright 2017 Google Inc. + * Copyright 2018 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From c1fa702e7c099af67357b5478bdde36183c2fa58 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Thu, 22 Feb 2018 18:10:01 -0500 Subject: [PATCH 5/5] Explain the empty default for a config value. --- src/configuration.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/configuration.cc b/src/configuration.cc index cc1b4d3e..53445e32 100644 --- a/src/configuration.cc +++ b/src/configuration.cc @@ -40,7 +40,8 @@ constexpr const int kMetadataIngestionDefaultRequestSizeLimitBytes = 8*1024*1024; constexpr const char kMetadataIngestionDefaultRawContentVersion[] = "0.1"; constexpr const int kInstanceUpdaterDefaultIntervalSeconds = 60*60; -constexpr const char kDefaultInstanceResourceType[] = ""; +constexpr const char kDefaultInstanceResourceType[] = + ""; // A blank value means "unspecified; detect via environment". constexpr const int kDockerUpdaterDefaultIntervalSeconds = 60; constexpr const char kDockerDefaultEndpointHost[] = "unix://%2Fvar%2Frun%2Fdocker.sock/";