From 1ccb224e2f55f941c641c5c26568ddd4b03b2be2 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Wed, 21 Mar 2018 21:05:47 -0400 Subject: [PATCH 1/9] Replace an std::bind instance missed in #74. --- src/api_server.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api_server.cc b/src/api_server.cc index c7b29dd3..18d252ee 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -155,7 +155,7 @@ MetadataReporter::MetadataReporter(MetadataAgent* agent, double period_s) environment_(agent->config_), auth_(environment_), period_(period_s), - reporter_thread_(&MetadataReporter::ReportMetadata, this) {} + reporter_thread_([=]() { ReportMetadata(); }) {} MetadataReporter::~MetadataReporter() { reporter_thread_.join(); From b2aed76af4881a4b289184a5496ef7364effb71b Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Wed, 21 Mar 2018 21:44:01 -0400 Subject: [PATCH 2/9] Factor out MetadataStore from MetadataAgent. --- src/api_server.cc | 75 +++++++++++++++++++++++++---------------------- src/api_server.h | 44 ++++++++++++++++++++------- src/updater.cc | 8 ++--- src/updater.h | 20 +++++++------ 4 files changed, 88 insertions(+), 59 deletions(-) diff --git a/src/api_server.cc b/src/api_server.cc index 18d252ee..9f97e404 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -36,10 +36,13 @@ namespace http = boost::network::http; namespace google { -MetadataAgent::Metadata MetadataAgent::Metadata::IGNORED() { - return MetadataAgent::Metadata(); +MetadataStore::Metadata MetadataStore::Metadata::IGNORED() { + return MetadataStore::Metadata(); } +MetadataStore::MetadataStore(const MetadataAgentConfiguration& config) + : config_(config) {} + class MetadataApiServer { public: @@ -57,7 +60,8 @@ class MetadataApiServer { std::shared_ptr conn); void log(const HttpServer::string_type& info); private: - const MetadataAgent& agent_; + const MetadataAgentConfiguration& config_; + const MetadataStore& store_; }; Handler handler_; @@ -76,10 +80,11 @@ class MetadataReporter { // Send the given set of metadata. void SendMetadata( - std::map&& metadata) + std::map&& metadata) throw (boost::system::system_error); - MetadataAgent* agent_; + const MetadataAgentConfiguration& config_; + MetadataStore* store_; Environment environment_; OAuth2 auth_; // The reporting period in seconds. @@ -88,14 +93,14 @@ class MetadataReporter { }; MetadataApiServer::Handler::Handler(const MetadataAgent& agent) - : agent_(agent) {} + : config_(agent.config()), store_(agent.store()) {} void MetadataApiServer::Handler::operator()(const HttpServer::request& request, std::shared_ptr conn) { static const std::string kPrefix = "/monitoredResource/"; // The format for the local metadata API request is: // {host}:{port}/monitoredResource/{id} - if (agent_.config_.VerboseLogging()) { + if (config_.VerboseLogging()) { LOG(INFO) << "Handler called: " << request.method << " " << request.destination << " headers: " << request.headers @@ -103,19 +108,19 @@ void MetadataApiServer::Handler::operator()(const HttpServer::request& request, } if (request.method == "GET" && request.destination.find(kPrefix) == 0) { std::string id = request.destination.substr(kPrefix.size()); - std::lock_guard lock(agent_.resource_mu_); - const auto result = agent_.resource_map_.find(id); - if (result == agent_.resource_map_.end()) { + std::lock_guard lock(store_.resource_mu_); + const auto result = store_.resource_map_.find(id); + if (result == store_.resource_map_.end()) { // TODO: This could be considered log spam. // As we add more resource mappings, these will become less and less // frequent, and could be promoted to ERROR. - if (agent_.config_.VerboseLogging()) { + if (config_.VerboseLogging()) { LOG(WARNING) << "No matching resource for " << id; } conn->set_status(HttpServer::connection::not_found); } else { const MonitoredResource& resource = result->second; - if (agent_.config_.VerboseLogging()) { + if (config_.VerboseLogging()) { LOG(INFO) << "Found resource for " << id << ": " << resource; } conn->set_status(HttpServer::connection::ok); @@ -151,8 +156,9 @@ MetadataApiServer::~MetadataApiServer() { } MetadataReporter::MetadataReporter(MetadataAgent* agent, double period_s) - : agent_(agent), - environment_(agent->config_), + : store_(agent->mutable_store()), + config_(agent->config()), + environment_(config_), auth_(environment_), period_(period_s), reporter_thread_([=]() { ReportMetadata(); }) {} @@ -168,16 +174,16 @@ void MetadataReporter::ReportMetadata() { std::this_thread::sleep_for(time::seconds(3)); // TODO: Do we need to be able to stop this? while (true) { - if (agent_->config_.VerboseLogging()) { + if (config_.VerboseLogging()) { LOG(INFO) << "Sending metadata request to server"; } try { - SendMetadata(agent_->GetMetadataMap()); - if (agent_->config_.VerboseLogging()) { + SendMetadata(store_->GetMetadataMap()); + if (config_.VerboseLogging()) { LOG(INFO) << "Metadata request sent successfully"; } - if (agent_->config_.MetadataReporterPurgeDeleted()) { - agent_->PurgeDeletedEntries(); + if (config_.MetadataReporterPurgeDeleted()) { + store_->PurgeDeletedEntries(); } } catch (const boost::system::system_error& e) { LOG(ERROR) << "Metadata request unsuccessful: " << e.what(); @@ -231,40 +237,39 @@ void SendMetadataRequest(std::vector&& entries, } void MetadataReporter::SendMetadata( - std::map&& metadata) + std::map&& metadata) throw (boost::system::system_error) { if (metadata.empty()) { - if (agent_->config_.VerboseLogging()) { + if (config_.VerboseLogging()) { LOG(INFO) << "No data to send"; } return; } - if (agent_->config_.VerboseLogging()) { + if (config_.VerboseLogging()) { LOG(INFO) << "Sending request to the server"; } const std::string project_id = environment_.NumericProjectId(); // The endpoint template is expected to be of the form // "https://stackdriver.googleapis.com/.../projects/{{project_id}}/...". const std::string endpoint = - format::Substitute(agent_->config_.MetadataIngestionEndpointFormat(), + format::Substitute(config_.MetadataIngestionEndpointFormat(), {{"project_id", project_id}}); const std::string auth_header = auth_.GetAuthHeaderValue(); - const std::string user_agent = agent_->config_.MetadataReporterUserAgent(); + const std::string user_agent = config_.MetadataReporterUserAgent(); const json::value empty_request = json::object({ {"entries", json::array({})}, }); const int empty_size = empty_request->ToString().size(); - const int limit_bytes = - agent_->config_.MetadataIngestionRequestSizeLimitBytes(); + const int limit_bytes = config_.MetadataIngestionRequestSizeLimitBytes(); int total_size = empty_size; std::vector entries; for (auto& entry : metadata) { const MonitoredResource& resource = entry.first; - MetadataAgent::Metadata& metadata = entry.second; + MetadataStore::Metadata& metadata = entry.second; if (metadata.ignore) { continue; } @@ -287,7 +292,7 @@ void MetadataReporter::SendMetadata( } if (total_size + size > limit_bytes) { SendMetadataRequest(std::move(entries), endpoint, auth_header, user_agent, - agent_->config_.VerboseLogging()); + config_.VerboseLogging()); entries.clear(); total_size = empty_size; } @@ -296,16 +301,16 @@ void MetadataReporter::SendMetadata( } if (!entries.empty()) { SendMetadataRequest(std::move(entries), endpoint, auth_header, user_agent, - agent_->config_.VerboseLogging()); + config_.VerboseLogging()); } } MetadataAgent::MetadataAgent(const MetadataAgentConfiguration& config) - : config_(config) {} + : config_(config), store_(config_) {} MetadataAgent::~MetadataAgent() {} -void MetadataAgent::UpdateResource(const std::vector& resource_ids, +void MetadataStore::UpdateResource(const std::vector& resource_ids, const MonitoredResource& resource) { std::lock_guard lock(resource_mu_); // TODO: How do we handle deleted resources? @@ -318,7 +323,7 @@ void MetadataAgent::UpdateResource(const std::vector& resource_ids, } } -void MetadataAgent::UpdateMetadata(const MonitoredResource& resource, +void MetadataStore::UpdateMetadata(const MonitoredResource& resource, Metadata&& entry) { std::lock_guard lock(metadata_mu_); if (config_.VerboseLogging()) { @@ -338,8 +343,8 @@ void MetadataAgent::UpdateMetadata(const MonitoredResource& resource, metadata_map_.emplace(resource, std::move(entry)); } -std::map - MetadataAgent::GetMetadataMap() const { +std::map + MetadataStore::GetMetadataMap() const { std::lock_guard lock(metadata_mu_); std::map result; @@ -351,7 +356,7 @@ std::map return result; } -void MetadataAgent::PurgeDeletedEntries() { +void MetadataStore::PurgeDeletedEntries() { std::lock_guard lock(metadata_mu_); for (auto it = metadata_map_.begin(); it != metadata_map_.end(); ) { diff --git a/src/api_server.h b/src/api_server.h index 5ef4e97f..91a6e852 100644 --- a/src/api_server.h +++ b/src/api_server.h @@ -40,8 +40,8 @@ class MetadataReporter; // A timestamp type. using Timestamp = time_point; -// Stores the metadata mapping and runs the metadata tasks. -class MetadataAgent { +// Stores the metadata mapping. +class MetadataStore { public: struct Metadata { Metadata(const std::string& version_, @@ -79,8 +79,7 @@ class MetadataAgent { metadata(json::object({})), ignore(true) {} }; - MetadataAgent(const MetadataAgentConfiguration& config); - ~MetadataAgent(); + MetadataStore(const MetadataAgentConfiguration& config); // Updates the local resource map entry for a given resource. // Each local id in `resource_ids` is effectively an alias for `resource`. @@ -93,13 +92,6 @@ class MetadataAgent { void UpdateMetadata(const MonitoredResource& resource, Metadata&& entry); - // Starts serving. - void start(); - - const MetadataAgentConfiguration& config() const { - return config_; - } - private: friend class MetadataApiServer; friend class MetadataReporter; @@ -117,6 +109,36 @@ class MetadataAgent { mutable std::mutex metadata_mu_; // A map from MonitoredResource to (JSON) resource metadata. std::map metadata_map_; +}; + +// Runs the metadata tasks. +class MetadataAgent { + public: + using Metadata = MetadataStore::Metadata; + + MetadataAgent(const MetadataAgentConfiguration& config); + ~MetadataAgent(); + + // Starts serving. + void start(); + + const MetadataAgentConfiguration& config() const { + return config_; + } + + const MetadataStore& store() const { + return store_; + } + + MetadataStore* mutable_store() { + return &store_; + } + + private: + const MetadataAgentConfiguration& config_; + + // The store for the metadata. + MetadataStore store_; // The Metadata API server. std::unique_ptr metadata_api_server_; diff --git a/src/updater.cc b/src/updater.cc index 0dd412cf..69e7abba 100644 --- a/src/updater.cc +++ b/src/updater.cc @@ -22,8 +22,8 @@ namespace google { -MetadataUpdater::MetadataUpdater(MetadataAgent* store, const std::string& name) - : store_(store), name_(name) {} +MetadataUpdater::MetadataUpdater(MetadataAgent* agent, const std::string& name) + : config_(agent->config()), store_(agent->mutable_store()), name_(name) {} MetadataUpdater::~MetadataUpdater() {} @@ -41,9 +41,9 @@ void MetadataUpdater::stop() { } PollingMetadataUpdater::PollingMetadataUpdater( - MetadataAgent* store, const std::string& name, double period_s, + MetadataAgent* agent, const std::string& name, double period_s, std::function()> query_metadata) - : MetadataUpdater(store, name), + : MetadataUpdater(agent, name), period_(period_s), query_metadata_(query_metadata), timer_(), diff --git a/src/updater.h b/src/updater.h index 84398b4f..409816df 100644 --- a/src/updater.h +++ b/src/updater.h @@ -36,23 +36,19 @@ class MetadataUpdater { struct ResourceMetadata { ResourceMetadata(const std::vector& ids_, const MonitoredResource& resource_, - MetadataAgent::Metadata&& metadata_) + MetadataStore::Metadata&& metadata_) : ids(ids_), resource(resource_), metadata(std::move(metadata_)) {} ResourceMetadata(ResourceMetadata&& other) : ResourceMetadata(other.ids, other.resource, std::move(other.metadata)) {} std::vector ids; MonitoredResource resource; - MetadataAgent::Metadata metadata; + MetadataStore::Metadata metadata; }; - MetadataUpdater(MetadataAgent* store, const std::string& name); + MetadataUpdater(MetadataAgent* agent, const std::string& name); virtual ~MetadataUpdater(); - const MetadataAgentConfiguration& config() { - return store_->config(); - } - // Starts updating. void start(); @@ -85,19 +81,25 @@ class MetadataUpdater { store_->UpdateMetadata(result.resource, std::move(result.metadata)); } + const MetadataAgentConfiguration& config() { + return config_; + } + private: // The name of the updater provided by subclasses. std::string name_; + const MetadataAgentConfiguration& config_; + // The store for the metadata. - MetadataAgent* store_; + MetadataStore* store_; }; // A class for all periodic updates of the metadata mapping. class PollingMetadataUpdater : public MetadataUpdater { public: PollingMetadataUpdater( - MetadataAgent* store, const std::string& name, double period_s, + MetadataAgent* agent, const std::string& name, double period_s, std::function()> query_metadata); ~PollingMetadataUpdater(); From 1561cfac575e6bf0c2cfb75dcff76ec89dc36dab Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Wed, 21 Mar 2018 22:30:24 -0400 Subject: [PATCH 3/9] Keep all MetadataStore functions together. --- src/api_server.cc | 144 +++++++++++++++++++++++----------------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/src/api_server.cc b/src/api_server.cc index 9f97e404..072a87f7 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -43,6 +43,78 @@ MetadataStore::Metadata MetadataStore::Metadata::IGNORED() { MetadataStore::MetadataStore(const MetadataAgentConfiguration& config) : config_(config) {} +void MetadataStore::UpdateResource(const std::vector& resource_ids, + const MonitoredResource& resource) { + std::lock_guard lock(resource_mu_); + // TODO: How do we handle deleted resources? + // TODO: Do we care if the value was already there? + for (const std::string& id : resource_ids) { + if (config_.VerboseLogging()) { + LOG(INFO) << "Updating resource map '" << id << "'->" << resource; + } + resource_map_.emplace(id, resource); + } +} + +void MetadataStore::UpdateMetadata(const MonitoredResource& resource, + Metadata&& entry) { + std::lock_guard lock(metadata_mu_); + if (config_.VerboseLogging()) { + LOG(INFO) << "Updating metadata map " << resource << "->{" + << "version: " << entry.version << ", " + << "is_deleted: " << entry.is_deleted << ", " + << "created_at: " << time::rfc3339::ToString(entry.created_at) << ", " + << "collected_at: " << time::rfc3339::ToString(entry.collected_at) + << ", " + << "metadata: " << *entry.metadata << ", " + << "ignore: " << entry.ignore + << "}"; + } + // Force value update. The repeated search is inefficient, but shouldn't + // be a huge deal. + metadata_map_.erase(resource); + metadata_map_.emplace(resource, std::move(entry)); +} + +std::map + MetadataStore::GetMetadataMap() const { + std::lock_guard lock(metadata_mu_); + + std::map result; + for (const auto& kv : metadata_map_) { + const MonitoredResource& resource = kv.first; + const Metadata& metadata = kv.second; + result.emplace(resource, metadata.Clone()); + } + return result; +} + +void MetadataStore::PurgeDeletedEntries() { + std::lock_guard lock(metadata_mu_); + + for (auto it = metadata_map_.begin(); it != metadata_map_.end(); ) { + const MonitoredResource& resource = it->first; + const Metadata& entry = it->second; + if (entry.is_deleted) { + if (config_.VerboseLogging()) { + LOG(INFO) << "Purging metadata entry " << resource << "->{" + << "version: " << entry.version << ", " + << "is_deleted: " << entry.is_deleted << ", " + << "created_at: " << time::rfc3339::ToString(entry.created_at) + << ", " + << "collected_at: " << time::rfc3339::ToString(entry.collected_at) + << ", " + << "metadata: " << *entry.metadata << ", " + << "ignore: " << entry.ignore + << "}"; + } + it = metadata_map_.erase(it); + } else { + ++it; + } + } +} + class MetadataApiServer { public: @@ -310,78 +382,6 @@ MetadataAgent::MetadataAgent(const MetadataAgentConfiguration& config) MetadataAgent::~MetadataAgent() {} -void MetadataStore::UpdateResource(const std::vector& resource_ids, - const MonitoredResource& resource) { - std::lock_guard lock(resource_mu_); - // TODO: How do we handle deleted resources? - // TODO: Do we care if the value was already there? - for (const std::string& id : resource_ids) { - if (config_.VerboseLogging()) { - LOG(INFO) << "Updating resource map '" << id << "'->" << resource; - } - resource_map_.emplace(id, resource); - } -} - -void MetadataStore::UpdateMetadata(const MonitoredResource& resource, - Metadata&& entry) { - std::lock_guard lock(metadata_mu_); - if (config_.VerboseLogging()) { - LOG(INFO) << "Updating metadata map " << resource << "->{" - << "version: " << entry.version << ", " - << "is_deleted: " << entry.is_deleted << ", " - << "created_at: " << time::rfc3339::ToString(entry.created_at) << ", " - << "collected_at: " << time::rfc3339::ToString(entry.collected_at) - << ", " - << "metadata: " << *entry.metadata << ", " - << "ignore: " << entry.ignore - << "}"; - } - // Force value update. The repeated search is inefficient, but shouldn't - // be a huge deal. - metadata_map_.erase(resource); - metadata_map_.emplace(resource, std::move(entry)); -} - -std::map - MetadataStore::GetMetadataMap() const { - std::lock_guard lock(metadata_mu_); - - std::map result; - for (const auto& kv : metadata_map_) { - const MonitoredResource& resource = kv.first; - const Metadata& metadata = kv.second; - result.emplace(resource, metadata.Clone()); - } - return result; -} - -void MetadataStore::PurgeDeletedEntries() { - std::lock_guard lock(metadata_mu_); - - for (auto it = metadata_map_.begin(); it != metadata_map_.end(); ) { - const MonitoredResource& resource = it->first; - const Metadata& entry = it->second; - if (entry.is_deleted) { - if (config_.VerboseLogging()) { - LOG(INFO) << "Purging metadata entry " << resource << "->{" - << "version: " << entry.version << ", " - << "is_deleted: " << entry.is_deleted << ", " - << "created_at: " << time::rfc3339::ToString(entry.created_at) - << ", " - << "collected_at: " << time::rfc3339::ToString(entry.collected_at) - << ", " - << "metadata: " << *entry.metadata << ", " - << "ignore: " << entry.ignore - << "}"; - } - it = metadata_map_.erase(it); - } else { - ++it; - } - } -} - void MetadataAgent::start() { metadata_api_server_.reset(new MetadataApiServer( *this, config_.MetadataApiNumThreads(), "0.0.0.0", From 8e9c078c2be98044e28937c5b68a3dc8c3a52ff4 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Wed, 21 Mar 2018 22:30:52 -0400 Subject: [PATCH 4/9] Use MetadataStore::Metadata instead of MetadataAgent::Metadata. --- src/api_server.h | 2 -- src/docker.cc | 4 ++-- src/instance.cc | 2 +- src/kubernetes.cc | 14 +++++++------- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/api_server.h b/src/api_server.h index 91a6e852..31d7e7ee 100644 --- a/src/api_server.h +++ b/src/api_server.h @@ -114,8 +114,6 @@ class MetadataStore { // Runs the metadata tasks. class MetadataAgent { public: - using Metadata = MetadataStore::Metadata; - MetadataAgent(const MetadataAgentConfiguration& config); ~MetadataAgent(); diff --git a/src/docker.cc b/src/docker.cc index 3c07b81c..9af81968 100644 --- a/src/docker.cc +++ b/src/docker.cc @@ -125,11 +125,11 @@ MetadataUpdater::ResourceMetadata DockerReader::GetContainerMetadata( std::vector{resource_id, resource_name}, resource, #ifdef ENABLE_DOCKER_METADATA - MetadataAgent::Metadata(config_.MetadataIngestionRawContentVersion(), + MetadataStore::Metadata(config_.MetadataIngestionRawContentVersion(), is_deleted, created_at, collected_at, std::move(raw_metadata)) #else - MetadataAgent::Metadata::IGNORED() + MetadataStore::Metadata::IGNORED() #endif ); } catch (const QueryException& e) { diff --git a/src/instance.cc b/src/instance.cc index 31a639e8..aa274470 100644 --- a/src/instance.cc +++ b/src/instance.cc @@ -54,7 +54,7 @@ std::vector std::vector{"", environment_.InstanceId()}, instance_resource, // TODO: Send actual instance metadata. - MetadataAgent::Metadata::IGNORED() + MetadataStore::Metadata::IGNORED() ); return result; } diff --git a/src/kubernetes.cc b/src/kubernetes.cc index bb315d12..22e06bd1 100644 --- a/src/kubernetes.cc +++ b/src/kubernetes.cc @@ -136,11 +136,11 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetNodeMetadata( std::vector{k8s_node_name}, k8s_node, #ifdef ENABLE_KUBERNETES_METADATA - MetadataAgent::Metadata(config_.MetadataIngestionRawContentVersion(), + MetadataStore::Metadata(config_.MetadataIngestionRawContentVersion(), is_deleted, created_at, collected_at, std::move(node_raw_metadata)) #else - MetadataAgent::Metadata::IGNORED() + MetadataStore::Metadata::IGNORED() #endif ); } @@ -238,11 +238,11 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetPodMetadata( std::vector{k8s_pod_id, k8s_pod_name}, k8s_pod, #ifdef ENABLE_KUBERNETES_METADATA - MetadataAgent::Metadata(config_.MetadataIngestionRawContentVersion(), + MetadataStore::Metadata(config_.MetadataIngestionRawContentVersion(), is_deleted, created_at, collected_at, std::move(pod_raw_metadata)) #else - MetadataAgent::Metadata::IGNORED() + MetadataStore::Metadata::IGNORED() #endif ); } @@ -346,11 +346,11 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetContainerMetadata( std::move(local_resource_ids), k8s_container, #ifdef ENABLE_KUBERNETES_METADATA - MetadataAgent::Metadata(kKubernetesApiVersion, + MetadataStore::Metadata(kKubernetesApiVersion, is_deleted, created_at, collected_at, std::move(container_raw_metadata)) #else - MetadataAgent::Metadata::IGNORED() + MetadataStore::Metadata::IGNORED() #endif ); } @@ -385,7 +385,7 @@ MetadataUpdater::ResourceMetadata KubernetesReader::GetLegacyResource( return MetadataUpdater::ResourceMetadata( std::vector{gke_container_pod_id, gke_container_name}, gke_container, - MetadataAgent::Metadata::IGNORED()); + MetadataStore::Metadata::IGNORED()); } std::vector From d6a9a88cc79778693e28b1560e6f22c494cca968 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Wed, 21 Mar 2018 23:05:41 -0400 Subject: [PATCH 5/9] Pull out MetadataStore into a separate file. --- src/Makefile | 1 + src/api_server.cc | 80 ---------------------------------- src/api_server.h | 82 +---------------------------------- src/store.cc | 103 +++++++++++++++++++++++++++++++++++++++++++ src/store.h | 108 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 213 insertions(+), 161 deletions(-) create mode 100644 src/store.cc create mode 100644 src/store.h diff --git a/src/Makefile b/src/Makefile index beb74a33..79b908ca 100644 --- a/src/Makefile +++ b/src/Makefile @@ -42,6 +42,7 @@ endif SRCS=\ metadatad.cc \ api_server.cc \ + store.cc \ configuration.cc \ updater.cc \ instance.cc \ diff --git a/src/api_server.cc b/src/api_server.cc index 072a87f7..cbe168bf 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -36,86 +36,6 @@ namespace http = boost::network::http; namespace google { -MetadataStore::Metadata MetadataStore::Metadata::IGNORED() { - return MetadataStore::Metadata(); -} - -MetadataStore::MetadataStore(const MetadataAgentConfiguration& config) - : config_(config) {} - -void MetadataStore::UpdateResource(const std::vector& resource_ids, - const MonitoredResource& resource) { - std::lock_guard lock(resource_mu_); - // TODO: How do we handle deleted resources? - // TODO: Do we care if the value was already there? - for (const std::string& id : resource_ids) { - if (config_.VerboseLogging()) { - LOG(INFO) << "Updating resource map '" << id << "'->" << resource; - } - resource_map_.emplace(id, resource); - } -} - -void MetadataStore::UpdateMetadata(const MonitoredResource& resource, - Metadata&& entry) { - std::lock_guard lock(metadata_mu_); - if (config_.VerboseLogging()) { - LOG(INFO) << "Updating metadata map " << resource << "->{" - << "version: " << entry.version << ", " - << "is_deleted: " << entry.is_deleted << ", " - << "created_at: " << time::rfc3339::ToString(entry.created_at) << ", " - << "collected_at: " << time::rfc3339::ToString(entry.collected_at) - << ", " - << "metadata: " << *entry.metadata << ", " - << "ignore: " << entry.ignore - << "}"; - } - // Force value update. The repeated search is inefficient, but shouldn't - // be a huge deal. - metadata_map_.erase(resource); - metadata_map_.emplace(resource, std::move(entry)); -} - -std::map - MetadataStore::GetMetadataMap() const { - std::lock_guard lock(metadata_mu_); - - std::map result; - for (const auto& kv : metadata_map_) { - const MonitoredResource& resource = kv.first; - const Metadata& metadata = kv.second; - result.emplace(resource, metadata.Clone()); - } - return result; -} - -void MetadataStore::PurgeDeletedEntries() { - std::lock_guard lock(metadata_mu_); - - for (auto it = metadata_map_.begin(); it != metadata_map_.end(); ) { - const MonitoredResource& resource = it->first; - const Metadata& entry = it->second; - if (entry.is_deleted) { - if (config_.VerboseLogging()) { - LOG(INFO) << "Purging metadata entry " << resource << "->{" - << "version: " << entry.version << ", " - << "is_deleted: " << entry.is_deleted << ", " - << "created_at: " << time::rfc3339::ToString(entry.created_at) - << ", " - << "collected_at: " << time::rfc3339::ToString(entry.collected_at) - << ", " - << "metadata: " << *entry.metadata << ", " - << "ignore: " << entry.ignore - << "}"; - } - it = metadata_map_.erase(it); - } else { - ++it; - } - } -} - - class MetadataApiServer { public: MetadataApiServer(const MetadataAgent& agent, int server_threads, diff --git a/src/api_server.h b/src/api_server.h index 31d7e7ee..e04dd619 100644 --- a/src/api_server.h +++ b/src/api_server.h @@ -18,16 +18,10 @@ //#include "config.h" -#include -#include #include -#include -#include #include "configuration.h" -#include "json.h" -#include "resource.h" -#include "time.h" +#include "store.h" namespace google { @@ -37,80 +31,6 @@ class MetadataApiServer; // A periodic reporter of metadata to Stackdriver. class MetadataReporter; -// A timestamp type. -using Timestamp = time_point; - -// Stores the metadata mapping. -class MetadataStore { - public: - struct Metadata { - Metadata(const std::string& version_, - bool is_deleted_, - const Timestamp& created_at_, - const Timestamp& collected_at_, - json::value metadata_) - : version(version_), is_deleted(is_deleted_), created_at(created_at_), - collected_at(collected_at_), metadata(std::move(metadata_)), - ignore(false) {} - Metadata(Metadata&& other) - : version(other.version), is_deleted(other.is_deleted), - created_at(other.created_at), collected_at(other.collected_at), - metadata(std::move(other.metadata)), ignore(other.ignore) {} - - Metadata Clone() const { - if (ignore) { - return {}; - } - return {version, is_deleted, created_at, collected_at, metadata->Clone()}; - } - - static Metadata IGNORED(); - - const std::string version; - const bool is_deleted; - const Timestamp created_at; - const Timestamp collected_at; - json::value metadata; - const bool ignore; - - private: - Metadata() - : version(), is_deleted(false), created_at(), collected_at(), - metadata(json::object({})), ignore(true) {} - }; - - MetadataStore(const MetadataAgentConfiguration& config); - - // Updates the local resource map entry for a given resource. - // Each local id in `resource_ids` is effectively an alias for `resource`. - // Adds a resource mapping from each of the `resource_ids` to the `resource`. - void UpdateResource(const std::vector& resource_ids, - const MonitoredResource& resource); - - // Updates metadata for a given resource. - // Adds a metadata mapping from the `resource` to the metadata `entry`. - void UpdateMetadata(const MonitoredResource& resource, - Metadata&& entry); - - private: - friend class MetadataApiServer; - friend class MetadataReporter; - - std::map GetMetadataMap() const; - void PurgeDeletedEntries(); - - const MetadataAgentConfiguration& config_; - - // A lock that guards access to the local resource map. - mutable std::mutex resource_mu_; - // A map from a locally unique id to MonitoredResource. - std::map resource_map_; - // A lock that guards access to the metadata map. - mutable std::mutex metadata_mu_; - // A map from MonitoredResource to (JSON) resource metadata. - std::map metadata_map_; -}; - // Runs the metadata tasks. class MetadataAgent { public: diff --git a/src/store.cc b/src/store.cc new file mode 100644 index 00000000..7d167fbc --- /dev/null +++ b/src/store.cc @@ -0,0 +1,103 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#include "store.h" + +#include "logging.h" +#include "time.h" + +namespace google { + +MetadataStore::Metadata MetadataStore::Metadata::IGNORED() { + return MetadataStore::Metadata(); +} + +MetadataStore::MetadataStore(const MetadataAgentConfiguration& config) + : config_(config) {} + +void MetadataStore::UpdateResource(const std::vector& resource_ids, + const MonitoredResource& resource) { + std::lock_guard lock(resource_mu_); + // TODO: How do we handle deleted resources? + // TODO: Do we care if the value was already there? + for (const std::string& id : resource_ids) { + if (config_.VerboseLogging()) { + LOG(INFO) << "Updating resource map '" << id << "'->" << resource; + } + resource_map_.emplace(id, resource); + } +} + +void MetadataStore::UpdateMetadata(const MonitoredResource& resource, + Metadata&& entry) { + std::lock_guard lock(metadata_mu_); + if (config_.VerboseLogging()) { + LOG(INFO) << "Updating metadata map " << resource << "->{" + << "version: " << entry.version << ", " + << "is_deleted: " << entry.is_deleted << ", " + << "created_at: " << time::rfc3339::ToString(entry.created_at) << ", " + << "collected_at: " << time::rfc3339::ToString(entry.collected_at) + << ", " + << "metadata: " << *entry.metadata << ", " + << "ignore: " << entry.ignore + << "}"; + } + // Force value update. The repeated search is inefficient, but shouldn't + // be a huge deal. + metadata_map_.erase(resource); + metadata_map_.emplace(resource, std::move(entry)); +} + +std::map + MetadataStore::GetMetadataMap() const { + std::lock_guard lock(metadata_mu_); + + std::map result; + for (const auto& kv : metadata_map_) { + const MonitoredResource& resource = kv.first; + const Metadata& metadata = kv.second; + result.emplace(resource, metadata.Clone()); + } + return result; +} + +void MetadataStore::PurgeDeletedEntries() { + std::lock_guard lock(metadata_mu_); + + for (auto it = metadata_map_.begin(); it != metadata_map_.end(); ) { + const MonitoredResource& resource = it->first; + const Metadata& entry = it->second; + if (entry.is_deleted) { + if (config_.VerboseLogging()) { + LOG(INFO) << "Purging metadata entry " << resource << "->{" + << "version: " << entry.version << ", " + << "is_deleted: " << entry.is_deleted << ", " + << "created_at: " << time::rfc3339::ToString(entry.created_at) + << ", " + << "collected_at: " << time::rfc3339::ToString(entry.collected_at) + << ", " + << "metadata: " << *entry.metadata << ", " + << "ignore: " << entry.ignore + << "}"; + } + it = metadata_map_.erase(it); + } else { + ++it; + } + } +} + +} diff --git a/src/store.h b/src/store.h new file mode 100644 index 00000000..6438c395 --- /dev/null +++ b/src/store.h @@ -0,0 +1,108 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ +#ifndef STORE_H_ +#define STORE_H_ + +//#include "config.h" + +#include +#include +#include + +#include "configuration.h" +#include "json.h" +#include "resource.h" +#include "time.h" + +namespace google { + +// A timestamp type. +using Timestamp = time_point; + +// Stores the metadata mapping. +class MetadataStore { + public: + struct Metadata { + Metadata(const std::string& version_, + bool is_deleted_, + const Timestamp& created_at_, + const Timestamp& collected_at_, + json::value metadata_) + : version(version_), is_deleted(is_deleted_), created_at(created_at_), + collected_at(collected_at_), metadata(std::move(metadata_)), + ignore(false) {} + Metadata(Metadata&& other) + : version(other.version), is_deleted(other.is_deleted), + created_at(other.created_at), collected_at(other.collected_at), + metadata(std::move(other.metadata)), ignore(other.ignore) {} + + Metadata Clone() const { + if (ignore) { + return {}; + } + return {version, is_deleted, created_at, collected_at, metadata->Clone()}; + } + + static Metadata IGNORED(); + + const std::string version; + const bool is_deleted; + const Timestamp created_at; + const Timestamp collected_at; + json::value metadata; + const bool ignore; + + private: + Metadata() + : version(), is_deleted(false), created_at(), collected_at(), + metadata(json::object({})), ignore(true) {} + }; + + MetadataStore(const MetadataAgentConfiguration& config); + + // Updates the local resource map entry for a given resource. + // Each local id in `resource_ids` is effectively an alias for `resource`. + // Adds a resource mapping from each of the `resource_ids` to the `resource`. + void UpdateResource(const std::vector& resource_ids, + const MonitoredResource& resource); + + // Updates metadata for a given resource. + // Adds a metadata mapping from the `resource` to the metadata `entry`. + void UpdateMetadata(const MonitoredResource& resource, + Metadata&& entry); + + private: + friend class MetadataApiServer; + friend class MetadataReporter; + + std::map GetMetadataMap() const; + void PurgeDeletedEntries(); + + const MetadataAgentConfiguration& config_; + + // A lock that guards access to the local resource map. + mutable std::mutex resource_mu_; + // A map from a locally unique id to MonitoredResource. + std::map resource_map_; + // A lock that guards access to the metadata map. + mutable std::mutex metadata_mu_; + // A map from MonitoredResource to (JSON) resource metadata. + std::map metadata_map_; +}; + +} + +#endif // STORE_H_ From e0338bb85be0ac918c8cc00a26470aa4ab0cec3b Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Wed, 21 Mar 2018 23:44:02 -0400 Subject: [PATCH 6/9] Pull out MetadataReporter into a separate file. --- src/Makefile | 1 + src/api_server.cc | 172 ------------------------------------------- src/api_server.h | 4 +- src/reporter.cc | 182 ++++++++++++++++++++++++++++++++++++++++++++++ src/reporter.h | 65 +++++++++++++++++ 5 files changed, 249 insertions(+), 175 deletions(-) create mode 100644 src/reporter.cc create mode 100644 src/reporter.h diff --git a/src/Makefile b/src/Makefile index 79b908ca..ef4b6937 100644 --- a/src/Makefile +++ b/src/Makefile @@ -43,6 +43,7 @@ SRCS=\ metadatad.cc \ api_server.cc \ store.cc \ + reporter.cc \ configuration.cc \ updater.cc \ instance.cc \ diff --git a/src/api_server.cc b/src/api_server.cc index cbe168bf..cac4b0ee 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -61,29 +61,6 @@ class MetadataApiServer { std::vector server_pool_; }; -class MetadataReporter { - public: - MetadataReporter(MetadataAgent* agent, double period_s); - ~MetadataReporter(); - - private: - // Metadata reporter. - void ReportMetadata(); - - // Send the given set of metadata. - void SendMetadata( - std::map&& metadata) - throw (boost::system::system_error); - - const MetadataAgentConfiguration& config_; - MetadataStore* store_; - Environment environment_; - OAuth2 auth_; - // The reporting period in seconds. - time::seconds period_; - std::thread reporter_thread_; -}; - MetadataApiServer::Handler::Handler(const MetadataAgent& agent) : config_(agent.config()), store_(agent.store()) {} @@ -147,155 +124,6 @@ MetadataApiServer::~MetadataApiServer() { } } -MetadataReporter::MetadataReporter(MetadataAgent* agent, double period_s) - : store_(agent->mutable_store()), - config_(agent->config()), - environment_(config_), - auth_(environment_), - period_(period_s), - reporter_thread_([=]() { ReportMetadata(); }) {} - -MetadataReporter::~MetadataReporter() { - reporter_thread_.join(); -} - -void MetadataReporter::ReportMetadata() { - LOG(INFO) << "Metadata reporter started"; - // Wait for the first collection to complete. - // TODO: Come up with a more robust synchronization mechanism. - std::this_thread::sleep_for(time::seconds(3)); - // TODO: Do we need to be able to stop this? - while (true) { - if (config_.VerboseLogging()) { - LOG(INFO) << "Sending metadata request to server"; - } - try { - SendMetadata(store_->GetMetadataMap()); - if (config_.VerboseLogging()) { - LOG(INFO) << "Metadata request sent successfully"; - } - if (config_.MetadataReporterPurgeDeleted()) { - store_->PurgeDeletedEntries(); - } - } catch (const boost::system::system_error& e) { - LOG(ERROR) << "Metadata request unsuccessful: " << e.what(); - } - std::this_thread::sleep_for(period_); - } - //LOG(INFO) << "Metadata reporter exiting"; -} - -namespace { - -void SendMetadataRequest(std::vector&& entries, - const std::string& endpoint, - const std::string& auth_header, - const std::string& user_agent, - bool verbose_logging) - throw (boost::system::system_error) { - json::value update_metadata_request = json::object({ - {"entries", json::array(std::move(entries))}, - }); - - if (verbose_logging) { - LOG(INFO) << "About to send request: POST " << endpoint - << " User-Agent: " << user_agent - << " " << *update_metadata_request; - } - - http::client client; - http::client::request request(endpoint); - std::string request_body = update_metadata_request->ToString(); - request << boost::network::header("User-Agent", user_agent); - request << boost::network::header("Content-Length", - std::to_string(request_body.size())); - request << boost::network::header("Content-Type", "application/json"); - request << boost::network::header("Authorization", auth_header); - request << boost::network::body(request_body); - http::client::response response = client.post(request); - if (status(response) >= 300) { - throw boost::system::system_error( - boost::system::errc::make_error_code(boost::system::errc::not_connected), - format::Substitute("Server responded with '{{message}}' ({{code}})", - {{"message", status_message(response)}, - {"code", format::str(status(response))}})); - } - if (verbose_logging) { - LOG(INFO) << "Server responded with " << body(response); - } - // TODO: process response. -} - -} - -void MetadataReporter::SendMetadata( - std::map&& metadata) - throw (boost::system::system_error) { - if (metadata.empty()) { - if (config_.VerboseLogging()) { - LOG(INFO) << "No data to send"; - } - return; - } - - if (config_.VerboseLogging()) { - LOG(INFO) << "Sending request to the server"; - } - const std::string project_id = environment_.NumericProjectId(); - // The endpoint template is expected to be of the form - // "https://stackdriver.googleapis.com/.../projects/{{project_id}}/...". - const std::string endpoint = - format::Substitute(config_.MetadataIngestionEndpointFormat(), - {{"project_id", project_id}}); - const std::string auth_header = auth_.GetAuthHeaderValue(); - const std::string user_agent = config_.MetadataReporterUserAgent(); - - const json::value empty_request = json::object({ - {"entries", json::array({})}, - }); - const int empty_size = empty_request->ToString().size(); - - const int limit_bytes = config_.MetadataIngestionRequestSizeLimitBytes(); - int total_size = empty_size; - - std::vector entries; - for (auto& entry : metadata) { - const MonitoredResource& resource = entry.first; - MetadataStore::Metadata& metadata = entry.second; - if (metadata.ignore) { - continue; - } - json::value metadata_entry = - json::object({ // MonitoredResourceMetadata - {"resource", resource.ToJSON()}, - {"rawContentVersion", json::string(metadata.version)}, - {"rawContent", std::move(metadata.metadata)}, - {"state", json::string(metadata.is_deleted ? "DELETED" : "ACTIVE")}, - {"createTime", json::string(time::rfc3339::ToString(metadata.created_at))}, - {"collectTime", json::string(time::rfc3339::ToString(metadata.collected_at))}, - }); - // TODO: This is probably all kinds of inefficient... - const int size = metadata_entry->ToString().size(); - if (empty_size + size > limit_bytes) { - LOG(ERROR) << "Individual entry too large: " << size - << "B is greater than the limit " << limit_bytes - << "B, dropping; entry " << *metadata_entry; - continue; - } - if (total_size + size > limit_bytes) { - SendMetadataRequest(std::move(entries), endpoint, auth_header, user_agent, - config_.VerboseLogging()); - entries.clear(); - total_size = empty_size; - } - entries.emplace_back(std::move(metadata_entry)); - total_size += size; - } - if (!entries.empty()) { - SendMetadataRequest(std::move(entries), endpoint, auth_header, user_agent, - config_.VerboseLogging()); - } -} MetadataAgent::MetadataAgent(const MetadataAgentConfiguration& config) : config_(config), store_(config_) {} diff --git a/src/api_server.h b/src/api_server.h index e04dd619..a6ef0212 100644 --- a/src/api_server.h +++ b/src/api_server.h @@ -21,6 +21,7 @@ #include #include "configuration.h" +#include "reporter.h" #include "store.h" namespace google { @@ -28,9 +29,6 @@ namespace google { // A server that implements the metadata agent API. class MetadataApiServer; -// A periodic reporter of metadata to Stackdriver. -class MetadataReporter; - // Runs the metadata tasks. class MetadataAgent { public: diff --git a/src/reporter.cc b/src/reporter.cc new file mode 100644 index 00000000..830c3b62 --- /dev/null +++ b/src/reporter.cc @@ -0,0 +1,182 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#include "reporter.h" + +#define BOOST_NETWORK_ENABLE_HTTPS +#include + +#include "api_server.h" +#include "format.h" +#include "json.h" +#include "logging.h" +#include "time.h" + +namespace http = boost::network::http; + +namespace google { + +MetadataReporter::MetadataReporter(MetadataAgent* agent, double period_s) + : store_(agent->mutable_store()), + config_(agent->config()), + environment_(config_), + auth_(environment_), + period_(period_s), + reporter_thread_([=]() { ReportMetadata(); }) {} + +MetadataReporter::~MetadataReporter() { + reporter_thread_.join(); +} + +void MetadataReporter::ReportMetadata() { + LOG(INFO) << "Metadata reporter started"; + // Wait for the first collection to complete. + // TODO: Come up with a more robust synchronization mechanism. + std::this_thread::sleep_for(time::seconds(3)); + // TODO: Do we need to be able to stop this? + while (true) { + if (config_.VerboseLogging()) { + LOG(INFO) << "Sending metadata request to server"; + } + try { + SendMetadata(store_->GetMetadataMap()); + if (config_.VerboseLogging()) { + LOG(INFO) << "Metadata request sent successfully"; + } + if (config_.MetadataReporterPurgeDeleted()) { + store_->PurgeDeletedEntries(); + } + } catch (const boost::system::system_error& e) { + LOG(ERROR) << "Metadata request unsuccessful: " << e.what(); + } + std::this_thread::sleep_for(period_); + } + //LOG(INFO) << "Metadata reporter exiting"; +} + +namespace { + +void SendMetadataRequest(std::vector&& entries, + const std::string& endpoint, + const std::string& auth_header, + const std::string& user_agent, + bool verbose_logging) + throw (boost::system::system_error) { + json::value update_metadata_request = json::object({ + {"entries", json::array(std::move(entries))}, + }); + + if (verbose_logging) { + LOG(INFO) << "About to send request: POST " << endpoint + << " User-Agent: " << user_agent + << " " << *update_metadata_request; + } + + http::client client; + http::client::request request(endpoint); + std::string request_body = update_metadata_request->ToString(); + request << boost::network::header("User-Agent", user_agent); + request << boost::network::header("Content-Length", + std::to_string(request_body.size())); + request << boost::network::header("Content-Type", "application/json"); + request << boost::network::header("Authorization", auth_header); + request << boost::network::body(request_body); + http::client::response response = client.post(request); + if (status(response) >= 300) { + throw boost::system::system_error( + boost::system::errc::make_error_code(boost::system::errc::not_connected), + format::Substitute("Server responded with '{{message}}' ({{code}})", + {{"message", status_message(response)}, + {"code", format::str(status(response))}})); + } + if (verbose_logging) { + LOG(INFO) << "Server responded with " << body(response); + } + // TODO: process response. +} + +} + +void MetadataReporter::SendMetadata( + std::map&& metadata) + throw (boost::system::system_error) { + if (metadata.empty()) { + if (config_.VerboseLogging()) { + LOG(INFO) << "No data to send"; + } + return; + } + + if (config_.VerboseLogging()) { + LOG(INFO) << "Sending request to the server"; + } + const std::string project_id = environment_.NumericProjectId(); + // The endpoint template is expected to be of the form + // "https://stackdriver.googleapis.com/.../projects/{{project_id}}/...". + const std::string endpoint = + format::Substitute(config_.MetadataIngestionEndpointFormat(), + {{"project_id", project_id}}); + const std::string auth_header = auth_.GetAuthHeaderValue(); + const std::string user_agent = config_.MetadataReporterUserAgent(); + + const json::value empty_request = json::object({ + {"entries", json::array({})}, + }); + const int empty_size = empty_request->ToString().size(); + + const int limit_bytes = config_.MetadataIngestionRequestSizeLimitBytes(); + int total_size = empty_size; + + std::vector entries; + for (auto& entry : metadata) { + const MonitoredResource& resource = entry.first; + MetadataStore::Metadata& metadata = entry.second; + if (metadata.ignore) { + continue; + } + json::value metadata_entry = + json::object({ // MonitoredResourceMetadata + {"resource", resource.ToJSON()}, + {"rawContentVersion", json::string(metadata.version)}, + {"rawContent", std::move(metadata.metadata)}, + {"state", json::string(metadata.is_deleted ? "DELETED" : "ACTIVE")}, + {"createTime", json::string(time::rfc3339::ToString(metadata.created_at))}, + {"collectTime", json::string(time::rfc3339::ToString(metadata.collected_at))}, + }); + // TODO: This is probably all kinds of inefficient... + const int size = metadata_entry->ToString().size(); + if (empty_size + size > limit_bytes) { + LOG(ERROR) << "Individual entry too large: " << size + << "B is greater than the limit " << limit_bytes + << "B, dropping; entry " << *metadata_entry; + continue; + } + if (total_size + size > limit_bytes) { + SendMetadataRequest(std::move(entries), endpoint, auth_header, user_agent, + config_.VerboseLogging()); + entries.clear(); + total_size = empty_size; + } + entries.emplace_back(std::move(metadata_entry)); + total_size += size; + } + if (!entries.empty()) { + SendMetadataRequest(std::move(entries), endpoint, auth_header, user_agent, + config_.VerboseLogging()); + } +} + +} diff --git a/src/reporter.h b/src/reporter.h new file mode 100644 index 00000000..c494e7a9 --- /dev/null +++ b/src/reporter.h @@ -0,0 +1,65 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ +#ifndef REPORTER_H_ +#define REPORTER_H_ + +//#include "config.h" + +#include +#include +#include +#include + +#include "configuration.h" +#include "environment.h" +#include "json.h" +#include "oauth2.h" +#include "resource.h" +#include "store.h" +#include "time.h" + +namespace google { + +// Storage for the metadata mapping. +class MetadataAgent; + +// A periodic reporter of metadata to Stackdriver. +class MetadataReporter { + public: + MetadataReporter(MetadataAgent* agent, double period_s); + ~MetadataReporter(); + + private: + // Metadata reporter. + void ReportMetadata(); + + // Send the given set of metadata. + void SendMetadata( + std::map&& metadata) + throw (boost::system::system_error); + + const MetadataAgentConfiguration& config_; + MetadataStore* store_; + Environment environment_; + OAuth2 auth_; + // The reporting period in seconds. + time::seconds period_; + std::thread reporter_thread_; +}; + +} + +#endif // REPORTER_H_ From 029b06f8f5f765eecbf814749376d7e2d8418372 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Thu, 22 Mar 2018 00:05:40 -0400 Subject: [PATCH 7/9] Pull out MetadataAgent into a separate file. --- src/Makefile | 1 + src/agent.cc | 36 +++++++++++++++++++++++++ src/agent.h | 67 +++++++++++++++++++++++++++++++++++++++++++++++ src/api_server.cc | 52 ------------------------------------ src/api_server.h | 58 +++++++++++++++++++--------------------- src/metadatad.cc | 2 +- src/reporter.cc | 2 +- src/updater.h | 2 +- 8 files changed, 134 insertions(+), 86 deletions(-) create mode 100644 src/agent.cc create mode 100644 src/agent.h diff --git a/src/Makefile b/src/Makefile index ef4b6937..56d81869 100644 --- a/src/Makefile +++ b/src/Makefile @@ -41,6 +41,7 @@ endif SRCS=\ metadatad.cc \ + agent.cc \ api_server.cc \ store.cc \ reporter.cc \ diff --git a/src/agent.cc b/src/agent.cc new file mode 100644 index 00000000..db26b94a --- /dev/null +++ b/src/agent.cc @@ -0,0 +1,36 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#include "agent.h" + +#include "api_server.h" + +namespace google { + +MetadataAgent::MetadataAgent(const MetadataAgentConfiguration& config) + : config_(config), store_(config_) {} + +MetadataAgent::~MetadataAgent() {} + +void MetadataAgent::start() { + metadata_api_server_.reset(new MetadataApiServer( + *this, config_.MetadataApiNumThreads(), "0.0.0.0", + config_.MetadataApiPort())); + reporter_.reset(new MetadataReporter( + this, config_.MetadataReporterIntervalSeconds())); +} + +} diff --git a/src/agent.h b/src/agent.h new file mode 100644 index 00000000..b8323e5b --- /dev/null +++ b/src/agent.h @@ -0,0 +1,67 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ +#ifndef AGENT_H_ +#define AGENT_H_ + +//#include "config.h" + +#include + +#include "configuration.h" +#include "reporter.h" +#include "store.h" + +namespace google { + +// A server that implements the metadata agent API. +class MetadataApiServer; + +// Runs the metadata tasks. +class MetadataAgent { + public: + MetadataAgent(const MetadataAgentConfiguration& config); + ~MetadataAgent(); + + // Starts serving. + void start(); + + const MetadataAgentConfiguration& config() const { + return config_; + } + + const MetadataStore& store() const { + return store_; + } + + MetadataStore* mutable_store() { + return &store_; + } + + private: + const MetadataAgentConfiguration& config_; + + // The store for the metadata. + MetadataStore store_; + + // The Metadata API server. + std::unique_ptr metadata_api_server_; + // The metadata reporter. + std::unique_ptr reporter_; +}; + +} + +#endif // AGENT_H_ diff --git a/src/api_server.cc b/src/api_server.cc index cac4b0ee..2e482384 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -16,51 +16,13 @@ #include "api_server.h" -#define BOOST_NETWORK_ENABLE_HTTPS -#include -#include #include -#include -#include -#include -#include "environment.h" -#include "format.h" #include "http_common.h" -#include "json.h" #include "logging.h" -#include "oauth2.h" -#include "time.h" - -namespace http = boost::network::http; namespace google { -class MetadataApiServer { - public: - MetadataApiServer(const MetadataAgent& agent, int server_threads, - const std::string& host, int port); - ~MetadataApiServer(); - - private: - class Handler; - using HttpServer = http::server; - class Handler { - public: - Handler(const MetadataAgent& agent); - void operator()(const HttpServer::request& request, - std::shared_ptr conn); - void log(const HttpServer::string_type& info); - private: - const MetadataAgentConfiguration& config_; - const MetadataStore& store_; - }; - - Handler handler_; - HttpServer server_; - std::vector server_pool_; -}; - MetadataApiServer::Handler::Handler(const MetadataAgent& agent) : config_(agent.config()), store_(agent.store()) {} @@ -124,18 +86,4 @@ MetadataApiServer::~MetadataApiServer() { } } - -MetadataAgent::MetadataAgent(const MetadataAgentConfiguration& config) - : config_(config), store_(config_) {} - -MetadataAgent::~MetadataAgent() {} - -void MetadataAgent::start() { - metadata_api_server_.reset(new MetadataApiServer( - *this, config_.MetadataApiNumThreads(), "0.0.0.0", - config_.MetadataApiPort())); - reporter_.reset(new MetadataReporter( - this, config_.MetadataReporterIntervalSeconds())); -} - } diff --git a/src/api_server.h b/src/api_server.h index a6ef0212..c36e8975 100644 --- a/src/api_server.h +++ b/src/api_server.h @@ -18,48 +18,44 @@ //#include "config.h" +#define BOOST_NETWORK_ENABLE_HTTPS +#include #include +#include +#include +#include "agent.h" #include "configuration.h" -#include "reporter.h" #include "store.h" +namespace http = boost::network::http; + namespace google { // A server that implements the metadata agent API. -class MetadataApiServer; - -// Runs the metadata tasks. -class MetadataAgent { +class MetadataApiServer { public: - MetadataAgent(const MetadataAgentConfiguration& config); - ~MetadataAgent(); - - // Starts serving. - void start(); - - const MetadataAgentConfiguration& config() const { - return config_; - } - - const MetadataStore& store() const { - return store_; - } - - MetadataStore* mutable_store() { - return &store_; - } + MetadataApiServer(const MetadataAgent& agent, int server_threads, + const std::string& host, int port); + ~MetadataApiServer(); private: - const MetadataAgentConfiguration& config_; - - // The store for the metadata. - MetadataStore store_; - - // The Metadata API server. - std::unique_ptr metadata_api_server_; - // The metadata reporter. - std::unique_ptr reporter_; + class Handler; + using HttpServer = http::server; + class Handler { + public: + Handler(const MetadataAgent& agent); + void operator()(const HttpServer::request& request, + std::shared_ptr conn); + void log(const HttpServer::string_type& info); + private: + const MetadataAgentConfiguration& config_; + const MetadataStore& store_; + }; + + Handler handler_; + HttpServer server_; + std::vector server_pool_; }; } diff --git a/src/metadatad.cc b/src/metadatad.cc index 03531cc5..4c1eb30b 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -16,7 +16,7 @@ //#include "config.h" -#include "api_server.h" +#include "agent.h" #include "configuration.h" #include "docker.h" #include "instance.h" diff --git a/src/reporter.cc b/src/reporter.cc index 830c3b62..b1541b02 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -19,7 +19,7 @@ #define BOOST_NETWORK_ENABLE_HTTPS #include -#include "api_server.h" +#include "agent.h" #include "format.h" #include "json.h" #include "logging.h" diff --git a/src/updater.h b/src/updater.h index 409816df..873f697a 100644 --- a/src/updater.h +++ b/src/updater.h @@ -24,7 +24,7 @@ #include #include -#include "api_server.h" +#include "agent.h" #include "resource.h" #include "time.h" From 715ab18fbe6bd0f6527c6180b945c54a691c9dcd Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Thu, 22 Mar 2018 17:04:57 -0400 Subject: [PATCH 8/9] Stop using MetadataAgent as a wrapper around config+store. Replace includes by forward declarations in headers whenever possible. --- src/agent.cc | 5 +++-- src/agent.h | 4 +++- src/api_server.cc | 12 ++++++++---- src/api_server.h | 16 ++++++++++------ src/docker.cc | 2 ++ src/docker.h | 16 +++++++++++----- src/instance.cc | 2 ++ src/instance.h | 16 +++++++++++----- src/kubernetes.cc | 11 +++++++---- src/kubernetes.h | 10 ++++++++-- src/metadatad.cc | 6 +++--- src/reporter.cc | 9 +++++---- src/reporter.h | 8 ++++---- src/store.cc | 1 + src/store.h | 4 +++- src/updater.cc | 11 +++++++---- src/updater.h | 11 ++++++++--- 17 files changed, 96 insertions(+), 48 deletions(-) diff --git a/src/agent.cc b/src/agent.cc index db26b94a..eb27d79a 100644 --- a/src/agent.cc +++ b/src/agent.cc @@ -17,6 +17,7 @@ #include "agent.h" #include "api_server.h" +#include "reporter.h" namespace google { @@ -27,10 +28,10 @@ MetadataAgent::~MetadataAgent() {} void MetadataAgent::start() { metadata_api_server_.reset(new MetadataApiServer( - *this, config_.MetadataApiNumThreads(), "0.0.0.0", + config_, store_, config_.MetadataApiNumThreads(), "0.0.0.0", config_.MetadataApiPort())); reporter_.reset(new MetadataReporter( - this, config_.MetadataReporterIntervalSeconds())); + config_, &store_, config_.MetadataReporterIntervalSeconds())); } } diff --git a/src/agent.h b/src/agent.h index b8323e5b..13cb3bed 100644 --- a/src/agent.h +++ b/src/agent.h @@ -21,7 +21,6 @@ #include #include "configuration.h" -#include "reporter.h" #include "store.h" namespace google { @@ -29,6 +28,9 @@ namespace google { // A server that implements the metadata agent API. class MetadataApiServer; +// A periodic reporter of metadata to Stackdriver. +class MetadataReporter; + // Runs the metadata tasks. class MetadataAgent { public: diff --git a/src/api_server.cc b/src/api_server.cc index 2e482384..f3f2f2d6 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -18,13 +18,16 @@ #include +#include "configuration.h" #include "http_common.h" #include "logging.h" +#include "store.h" namespace google { -MetadataApiServer::Handler::Handler(const MetadataAgent& agent) - : config_(agent.config()), store_(agent.store()) {} +MetadataApiServer::Handler::Handler(const MetadataAgentConfiguration& config, + const MetadataStore& store) + : config_(config), store_(store) {} void MetadataApiServer::Handler::operator()(const HttpServer::request& request, std::shared_ptr conn) { @@ -65,10 +68,11 @@ void MetadataApiServer::Handler::log(const HttpServer::string_type& info) { } -MetadataApiServer::MetadataApiServer(const MetadataAgent& agent, +MetadataApiServer::MetadataApiServer(const MetadataAgentConfiguration& config, + const MetadataStore& store, int server_threads, const std::string& host, int port) - : handler_(agent), + : handler_(config, store), server_( HttpServer::options(handler_) .address(host) diff --git a/src/api_server.h b/src/api_server.h index c36e8975..1ffce7bb 100644 --- a/src/api_server.h +++ b/src/api_server.h @@ -24,18 +24,21 @@ #include #include -#include "agent.h" -#include "configuration.h" -#include "store.h" - namespace http = boost::network::http; namespace google { +// Configuration object. +class MetadataAgentConfiguration; + +// Storage for the metadata mapping. +class MetadataStore; + // A server that implements the metadata agent API. class MetadataApiServer { public: - MetadataApiServer(const MetadataAgent& agent, int server_threads, + MetadataApiServer(const MetadataAgentConfiguration& config, + const MetadataStore& store, int server_threads, const std::string& host, int port); ~MetadataApiServer(); @@ -44,7 +47,8 @@ class MetadataApiServer { using HttpServer = http::server; class Handler { public: - Handler(const MetadataAgent& agent); + Handler(const MetadataAgentConfiguration& config, + const MetadataStore& store); void operator()(const HttpServer::request& request, std::shared_ptr conn); void log(const HttpServer::string_type& info); diff --git a/src/docker.cc b/src/docker.cc index 9af81968..73770ac7 100644 --- a/src/docker.cc +++ b/src/docker.cc @@ -21,11 +21,13 @@ #include #include +#include "configuration.h" #include "format.h" #include "instance.h" #include "json.h" #include "logging.h" #include "resource.h" +#include "store.h" #include "time.h" namespace http = boost::network::http; diff --git a/src/docker.h b/src/docker.h index 46c9de5f..40cbf64d 100644 --- a/src/docker.h +++ b/src/docker.h @@ -20,12 +20,17 @@ #include -#include "configuration.h" #include "environment.h" #include "updater.h" namespace google { +// Configuration object. +class MetadataAgentConfiguration; + +// Storage for the metadata mapping. +class MetadataStore; + class DockerReader { public: DockerReader(const MetadataAgentConfiguration& config); @@ -62,10 +67,11 @@ class DockerReader { class DockerUpdater : public PollingMetadataUpdater { public: - DockerUpdater(MetadataAgent* server) - : reader_(server->config()), PollingMetadataUpdater( - server, "DockerUpdater", - server->config().DockerUpdaterIntervalSeconds(), + DockerUpdater(const MetadataAgentConfiguration& config, + MetadataStore* store) + : reader_(config), PollingMetadataUpdater( + config, store, "DockerUpdater", + config.DockerUpdaterIntervalSeconds(), [=]() { return reader_.MetadataQuery(); }) { } protected: diff --git a/src/instance.cc b/src/instance.cc index aa274470..218797d3 100644 --- a/src/instance.cc +++ b/src/instance.cc @@ -19,8 +19,10 @@ #include #include +#include "configuration.h" #include "json.h" #include "logging.h" +#include "store.h" #include "time.h" namespace http = boost::network::http; diff --git a/src/instance.h b/src/instance.h index 846f2721..dc24e337 100644 --- a/src/instance.h +++ b/src/instance.h @@ -20,13 +20,18 @@ #include -#include "configuration.h" #include "environment.h" #include "resource.h" #include "updater.h" namespace google { +// Configuration object. +class MetadataAgentConfiguration; + +// Storage for the metadata mapping. +class MetadataStore; + class InstanceReader { public: InstanceReader(const MetadataAgentConfiguration& config); @@ -43,10 +48,11 @@ class InstanceReader { class InstanceUpdater : public PollingMetadataUpdater { public: - InstanceUpdater(MetadataAgent* server) - : reader_(server->config()), PollingMetadataUpdater( - server, "InstanceUpdater", - server->config().InstanceUpdaterIntervalSeconds(), + InstanceUpdater(const MetadataAgentConfiguration& config, + MetadataStore* store) + : reader_(config), PollingMetadataUpdater( + config, store, "InstanceUpdater", + config.InstanceUpdaterIntervalSeconds(), [=]() { return reader_.MetadataQuery(); }) { } private: InstanceReader reader_; diff --git a/src/kubernetes.cc b/src/kubernetes.cc index 22e06bd1..862e1149 100644 --- a/src/kubernetes.cc +++ b/src/kubernetes.cc @@ -27,12 +27,14 @@ #include #include +#include "configuration.h" #include "format.h" #include "http_common.h" #include "instance.h" #include "json.h" #include "logging.h" #include "resource.h" +#include "store.h" #include "time.h" namespace http = boost::network::http; @@ -1158,10 +1160,11 @@ void KubernetesReader::WatchNodes( LOG(INFO) << "Watch thread (node) exiting"; } -KubernetesUpdater::KubernetesUpdater(MetadataAgent* server) - : reader_(server->config()), PollingMetadataUpdater( - server, "KubernetesUpdater", - server->config().KubernetesUpdaterIntervalSeconds(), +KubernetesUpdater::KubernetesUpdater(const MetadataAgentConfiguration& config, + MetadataStore* store) + : reader_(config), PollingMetadataUpdater( + config, store, "KubernetesUpdater", + config.KubernetesUpdaterIntervalSeconds(), [=]() { return reader_.MetadataQuery(); }) { } bool KubernetesUpdater::ValidateConfiguration() const { diff --git a/src/kubernetes.h b/src/kubernetes.h index 91378a0e..39b35d35 100644 --- a/src/kubernetes.h +++ b/src/kubernetes.h @@ -25,13 +25,18 @@ #include #include -#include "configuration.h" #include "environment.h" #include "json.h" #include "updater.h" namespace google { +// Configuration object. +class MetadataAgentConfiguration; + +// Storage for the metadata mapping. +class MetadataStore; + class KubernetesReader { public: KubernetesReader(const MetadataAgentConfiguration& config); @@ -153,7 +158,8 @@ class KubernetesReader { class KubernetesUpdater : public PollingMetadataUpdater { public: - KubernetesUpdater(MetadataAgent* server); + KubernetesUpdater(const MetadataAgentConfiguration& config, + MetadataStore* store); ~KubernetesUpdater() { if (node_watch_thread_.joinable()) { node_watch_thread_.join(); diff --git a/src/metadatad.cc b/src/metadatad.cc index 4c1eb30b..7fd17d09 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -31,9 +31,9 @@ int main(int ac, char** av) { google::MetadataAgent server(config); - google::InstanceUpdater instance_updater(&server); - google::DockerUpdater docker_updater(&server); - google::KubernetesUpdater kubernetes_updater(&server); + google::InstanceUpdater instance_updater(config, server.mutable_store()); + google::DockerUpdater docker_updater(config, server.mutable_store()); + google::KubernetesUpdater kubernetes_updater(config, server.mutable_store()); instance_updater.start(); docker_updater.start(); diff --git a/src/reporter.cc b/src/reporter.cc index b1541b02..4bc74c2b 100644 --- a/src/reporter.cc +++ b/src/reporter.cc @@ -19,7 +19,7 @@ #define BOOST_NETWORK_ENABLE_HTTPS #include -#include "agent.h" +#include "configuration.h" #include "format.h" #include "json.h" #include "logging.h" @@ -29,9 +29,10 @@ namespace http = boost::network::http; namespace google { -MetadataReporter::MetadataReporter(MetadataAgent* agent, double period_s) - : store_(agent->mutable_store()), - config_(agent->config()), +MetadataReporter::MetadataReporter(const MetadataAgentConfiguration& config, + MetadataStore* store, double period_s) + : store_(store), + config_(config), environment_(config_), auth_(environment_), period_(period_s), diff --git a/src/reporter.h b/src/reporter.h index c494e7a9..dc8c4731 100644 --- a/src/reporter.h +++ b/src/reporter.h @@ -23,7 +23,6 @@ #include #include -#include "configuration.h" #include "environment.h" #include "json.h" #include "oauth2.h" @@ -33,13 +32,14 @@ namespace google { -// Storage for the metadata mapping. -class MetadataAgent; +// Configuration object. +class MetadataAgentConfiguration; // A periodic reporter of metadata to Stackdriver. class MetadataReporter { public: - MetadataReporter(MetadataAgent* agent, double period_s); + MetadataReporter(const MetadataAgentConfiguration& config, + MetadataStore* store, double period_s); ~MetadataReporter(); private: diff --git a/src/store.cc b/src/store.cc index 7d167fbc..32a36dd7 100644 --- a/src/store.cc +++ b/src/store.cc @@ -16,6 +16,7 @@ #include "store.h" +#include "configuration.h" #include "logging.h" #include "time.h" diff --git a/src/store.h b/src/store.h index 6438c395..211adace 100644 --- a/src/store.h +++ b/src/store.h @@ -22,13 +22,15 @@ #include #include -#include "configuration.h" #include "json.h" #include "resource.h" #include "time.h" namespace google { +// Configuration object. +class MetadataAgentConfiguration; + // A timestamp type. using Timestamp = time_point; diff --git a/src/updater.cc b/src/updater.cc index 69e7abba..24c535bf 100644 --- a/src/updater.cc +++ b/src/updater.cc @@ -18,12 +18,14 @@ #include +#include "configuration.h" #include "logging.h" namespace google { -MetadataUpdater::MetadataUpdater(MetadataAgent* agent, const std::string& name) - : config_(agent->config()), store_(agent->mutable_store()), name_(name) {} +MetadataUpdater::MetadataUpdater(const MetadataAgentConfiguration& config, + MetadataStore* store, const std::string& name) + : config_(config), store_(store), name_(name) {} MetadataUpdater::~MetadataUpdater() {} @@ -41,9 +43,10 @@ void MetadataUpdater::stop() { } PollingMetadataUpdater::PollingMetadataUpdater( - MetadataAgent* agent, const std::string& name, double period_s, + const MetadataAgentConfiguration& config, MetadataStore* store, + const std::string& name, double period_s, std::function()> query_metadata) - : MetadataUpdater(agent, name), + : MetadataUpdater(config, store, name), period_(period_s), query_metadata_(query_metadata), timer_(), diff --git a/src/updater.h b/src/updater.h index 873f697a..62a8534f 100644 --- a/src/updater.h +++ b/src/updater.h @@ -24,12 +24,15 @@ #include #include -#include "agent.h" #include "resource.h" +#include "store.h" #include "time.h" namespace google { +// Configuration object. +class MetadataAgentConfiguration; + // An abstract class for asynchronous updates of the metadata mapping. class MetadataUpdater { public: @@ -46,7 +49,8 @@ class MetadataUpdater { MetadataStore::Metadata metadata; }; - MetadataUpdater(MetadataAgent* agent, const std::string& name); + MetadataUpdater(const MetadataAgentConfiguration& config, + MetadataStore* store, const std::string& name); virtual ~MetadataUpdater(); // Starts updating. @@ -99,7 +103,8 @@ class MetadataUpdater { class PollingMetadataUpdater : public MetadataUpdater { public: PollingMetadataUpdater( - MetadataAgent* agent, const std::string& name, double period_s, + const MetadataAgentConfiguration& config, MetadataStore* store, + const std::string& name, double period_s, std::function()> query_metadata); ~PollingMetadataUpdater(); From a9e65ad79791fa2a4702b83f8ab97c31ef07e4d0 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Fri, 23 Mar 2018 13:17:14 -0400 Subject: [PATCH 9/9] Forward-declare MetadataAgentConfiguration in agent.h. --- src/agent.cc | 1 + src/agent.h | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/agent.cc b/src/agent.cc index eb27d79a..0eb536f9 100644 --- a/src/agent.cc +++ b/src/agent.cc @@ -16,6 +16,7 @@ #include "agent.h" +#include "configuration.h" #include "api_server.h" #include "reporter.h" diff --git a/src/agent.h b/src/agent.h index 13cb3bed..852864e1 100644 --- a/src/agent.h +++ b/src/agent.h @@ -20,11 +20,13 @@ #include -#include "configuration.h" #include "store.h" namespace google { +// Configuration object. +class MetadataAgentConfiguration; + // A server that implements the metadata agent API. class MetadataApiServer;