Guard most logging with the verbose logging option

README.md

@@ -4,7 +4,8 @@ This is the Stackdriver metadata agent.
 
 1. Install runtime dependencies:
 
-       $ sudo apt-get install libyajl2 libboost-system1.55.0 libboost-thread1.55.0
+       $ sudo apt-get install libyajl2 libboost-program-options1.55.0 \
+         libboost-system1.55.0 libboost-thread1.55.0
 
 2. Install build dependencies:
 
@@ -18,7 +19,8 @@ This is the Stackdriver metadata agent.
 
 1. Install runtime dependencies (Ubuntu 16.04 special edition):
 
-       $ sudo apt-get install libssl1.0.0 libyajl2 libboost-system1.58.0 \
+       $ sudo apt-get install libssl1.0.0 libyajl2 \
+         libboost-program-options1.58.0 libboost-system1.58.0 \
          libboost-thread1.58.0
 
 2. Install build dependencies (Ubuntu 16.04 special edition):

pkg/deb/debian/control.base

@@ -6,7 +6,7 @@ Build-Depends: g++, cmake, dpkg-dev, libyajl-dev (>= 2.0), libssl-dev, libboost1
 
 Package: stackdriver-metadata
 Architecture: amd64
-Depends: libyajl2, libboost-system1.55.0, libboost-thread1.55.0
+Depends: libyajl2, libboost-program-options1.55.0 libboost-system1.55.0, libboost-thread1.55.0
 Description: Stackdriver metadata collection daemon
   The Stackdriver metadata daemon collects resource metadata and
   sends it to the Stackdriver service.

pkg/deb/debian/control.xenial

@@ -6,7 +6,7 @@ Build-Depends: g++, cmake, dpkg-dev, libyajl-dev (>= 2.0), libssl-dev, libboost1
 
 Package: stackdriver-metadata
 Architecture: amd64
-Depends: libssl1.0.0, libyajl2, libboost-system1.58.0, libboost-thread1.58.0
+Depends: libssl1.0.0, libyajl2, libboost-program-options1.58.0, libboost-system1.58.0, libboost-thread1.58.0
 Description: Stackdriver metadata collection daemon
   The Stackdriver metadata daemon collects resource metadata and
   sends it to the Stackdriver service.

src/Makefile

@@ -8,8 +8,8 @@ SED_I=/usr/bin/env sed -i
 CMAKE=cmake
 CXXFLAGS=-std=c++11 -g -DENABLE_KUBERNETES_METADATA -I$(CPP_NETLIB_DIR) -I$(YAML_CPP_DIR)/include
 LDFLAGS=-L$(CPP_NETLIB_LIBDIR) -L$(YAML_CPP_LIBDIR)
-LDLIBS=-lcppnetlib-uri -lcppnetlib-client-connections -lboost_system \
-    -lboost_thread -lpthread -lyajl -lssl -lcrypto -lyaml-cpp
+LDLIBS=-lcppnetlib-uri -lcppnetlib-client-connections -lboost_program_options \
+    -lboost_system -lboost_thread -lpthread -lyajl -lssl -lcrypto -lyaml-cpp
 SED_EXTRA=
 
 UNAME_S=$(shell uname -s)

src/api_server.cc

@@ -109,23 +109,29 @@ void MetadataApiServer::Handler::operator()(const HttpServer::request& request,
   static const std::string kPrefix = "/monitoredResource/";
   // The format for the local metadata API request is:
   //   {host}:{port}/monitoredResource/{id}
-  LOG(INFO) << "Handler called: " << request.method
-            << " " << request.destination
-            << " headers: " << request.headers
-            << " body: " << request.body;
+  if (agent_.config_.VerboseLogging()) {
+    LOG(INFO) << "Handler called: " << request.method
+              << " " << request.destination
+              << " headers: " << request.headers
+              << " body: " << request.body;
+  }
   if (request.method == "GET" && request.destination.find(kPrefix) == 0) {
     std::string id = request.destination.substr(kPrefix.size());
     const auto result = agent_.resource_map_.find(id);
     if (result == agent_.resource_map_.end()) {
       // TODO: This could be considered log spam.
       // As we add more resource mappings, these will become less and less
       // frequent, and could be promoted to ERROR.
-      LOG(WARNING) << "No matching resource for " << id;
+      if (agent_.config_.VerboseLogging()) {
+        LOG(WARNING) << "No matching resource for " << id;
+      }
       response = HttpServer::response::stock_reply(
           HttpServer::response::not_found, "");
     } else {
       const MonitoredResource& resource = result->second;
-      LOG(INFO) << "Found resource for " << id << ": " << resource;
+      if (agent_.config_.VerboseLogging()) {
+        LOG(INFO) << "Found resource for " << id << ": " << resource;
+      }
       response = HttpServer::response::stock_reply(
           HttpServer::response::ok, resource.ToJSON()->ToString());
     }
@@ -176,9 +182,13 @@ void MetadataReporter::ReportMetadata() {
   std::this_thread::sleep_for(std::chrono::seconds(3));
   // TODO: Do we need to be able to stop this?
   while (true) {
-    LOG(INFO) << "Sending metadata request to server";
+    if (agent_.config_.VerboseLogging()) {
+      LOG(INFO) << "Sending metadata request to server";
+    }
     SendMetadata(agent_.GetMetadataMap());
-    LOG(INFO) << "Metadata request sent successfully";
+    if (agent_.config_.VerboseLogging()) {
+      LOG(INFO) << "Metadata request sent successfully";
+    }
     std::this_thread::sleep_for(period_);
   }
   //LOG(INFO) << "Metadata reporter exiting";
@@ -188,13 +198,16 @@ namespace {
 
 void SendMetadataRequest(std::vector<json::value>&& entries,
                          const std::string& endpoint,
-                         const std::string& auth_header) {
+                         const std::string& auth_header,
+                         bool verbose_logging) {
   json::value update_metadata_request = json::object({
     {"entries", json::array(std::move(entries))},
   });
 
-  LOG(INFO) << "About to send request: POST " << endpoint
-            << " " << *update_metadata_request;
+  if (verbose_logging) {
+    LOG(INFO) << "About to send request: POST " << endpoint
+              << " " << *update_metadata_request;
+  }
 
   http::client client;
   http::client::request request(endpoint);
@@ -205,7 +218,9 @@ void SendMetadataRequest(std::vector<json::value>&& entries,
   request << boost::network::header("Authorization", auth_header);
   request << boost::network::body(request_body);
   http::client::response response = client.post(request);
-  LOG(INFO) << "Server responded with " << body(response);
+  if (verbose_logging) {
+    LOG(INFO) << "Server responded with " << body(response);
+  }
   // TODO: process response.
 }
 
@@ -214,11 +229,15 @@ void SendMetadataRequest(std::vector<json::value>&& entries,
 void MetadataReporter::SendMetadata(
     std::map<MonitoredResource, MetadataAgent::Metadata>&& metadata) {
   if (metadata.empty()) {
-    LOG(INFO) << "No data to send";
+    if (agent_.config_.VerboseLogging()) {
+      LOG(INFO) << "No data to send";
+    }
     return;
   }
 
-  LOG(INFO) << "Sending request to the server";
+  if (agent_.config_.VerboseLogging()) {
+    LOG(INFO) << "Sending request to the server";
+  }
   const std::string project_id = environment_.NumericProjectId();
   // The endpoint template is expected to be of the form
   // "https://stackdriver.googleapis.com/.../projects/{{project_id}}/...".
@@ -261,15 +280,17 @@ void MetadataReporter::SendMetadata(
       continue;
     }
     if (total_size + size > limit_bytes) {
-      SendMetadataRequest(std::move(entries), endpoint, auth_header);
+      SendMetadataRequest(std::move(entries), endpoint, auth_header,
+                          agent_.config_.VerboseLogging());
       entries.clear();
       total_size = empty_size;
     }
     entries.emplace_back(std::move(metadata_entry));
     total_size += size;
   }
   if (!entries.empty()) {
-    SendMetadataRequest(std::move(entries), endpoint, auth_header);
+    SendMetadataRequest(std::move(entries), endpoint, auth_header,
+                        agent_.config_.VerboseLogging());
   }
 }
 
@@ -285,17 +306,22 @@ void MetadataAgent::UpdateResource(const std::vector<std::string>& resource_ids,
   // TODO: How do we handle deleted resources?
   // TODO: Do we care if the value was already there?
   for (const std::string& id : resource_ids) {
-    LOG(INFO) << "Updating resource map '" << id << "'->" << resource;
+    if (config_.VerboseLogging()) {
+      LOG(INFO) << "Updating resource map '" << id << "'->" << resource;
+    }
     resource_map_.emplace(id, resource);
   }
-  LOG(INFO) << "Updating metadata map " << resource << "->{"
-            << "version: " << entry.version << ", "
-            << "is_deleted: " << entry.is_deleted << ", "
-            << "created_at: " << rfc3339::ToString(entry.created_at) << ", "
-            << "collected_at: " << rfc3339::ToString(entry.collected_at) << ", "
-            << "metadata: " << *entry.metadata << ", "
-            << "ignore: " << entry.ignore
-            << "}";
+  if (config_.VerboseLogging()) {
+    LOG(INFO) << "Updating metadata map " << resource << "->{"
+              << "version: " << entry.version << ", "
+              << "is_deleted: " << entry.is_deleted << ", "
+              << "created_at: " << rfc3339::ToString(entry.created_at) << ", "
+              << "collected_at: " << rfc3339::ToString(entry.collected_at)
+              << ", "
+              << "metadata: " << *entry.metadata << ", "
+              << "ignore: " << entry.ignore
+              << "}";
+  }
   // Force value update. The repeated search is inefficient, but shouldn't
   // be a huge deal.
   metadata_map_.erase(resource);

src/api_server.h

@@ -92,6 +92,10 @@ class MetadataAgent {
   // Starts serving.
   void start();
 
+  const MetadataAgentConfiguration& config() const {
+    return config_;
+  }
+
  private:
   friend class MetadataApiServer;
   friend class MetadataReporter;

src/configuration.cc

@@ -16,13 +16,16 @@
 
 #include "configuration.h"
 
+#include <boost/program_options.hpp>
+#include <iostream>
 #include <map>
 
 #include <yaml-cpp/yaml.h>
 
 namespace google {
 
 namespace {
+constexpr const char kConfigFileFlag[] = "config-file";
 
 constexpr const char kDefaultProjectId[] = "";
 constexpr const char kDefaultCredentialsFile[] = "";
@@ -51,6 +54,7 @@ constexpr const char kDefaultInstanceZone[] = "";
 MetadataAgentConfiguration::MetadataAgentConfiguration()
     : project_id_(kDefaultProjectId),
       credentials_file_(kDefaultCredentialsFile),
+      verbose_logging_(false),
       metadata_api_num_threads_(kMetadataApiDefaultNumThreads),
       metadata_api_port_(kMetadataApiDefaultPort),
       metadata_reporter_interval_seconds_(
@@ -71,9 +75,41 @@ MetadataAgentConfiguration::MetadataAgentConfiguration()
       instance_id_(kDefaultInstanceId),
       instance_zone_(kDefaultInstanceZone) {}
 
-MetadataAgentConfiguration::MetadataAgentConfiguration(
-    const std::string& filename) : MetadataAgentConfiguration()
-{
+int MetadataAgentConfiguration::ParseArguments(int ac, char** av) {
+  std::string config_file;
+  boost::program_options::options_description flags_desc;
+  flags_desc.add_options()
+      ("help,h", "Print help message")
+      ("verbose,v", boost::program_options::bool_switch(&verbose_logging_),
+           "Enable verbose logging")
+      ;
+  boost::program_options::options_description hidden_desc;
+  hidden_desc.add_options()
+      (kConfigFileFlag,
+           boost::program_options::value<std::string>(&config_file)
+               ->default_value(""),
+           "Configuration file location")
+      ;
+  boost::program_options::options_description all_desc;
+  all_desc.add(flags_desc).add(hidden_desc);
+  boost::program_options::positional_options_description positional_desc;
+  positional_desc.add(kConfigFileFlag, 1);
+  boost::program_options::variables_map flags;
+  boost::program_options::store(
+      boost::program_options::command_line_parser(ac, av)
+          .options(all_desc).positional(positional_desc).run(), flags);
+  boost::program_options::notify(flags);
+
+  if (flags.count("help")) {
+    std::cout << flags_desc << std::endl;
+    return 1;
+  }
+
+  ParseConfigFile(config_file);
+  return 0;
+}
+
+void MetadataAgentConfiguration::ParseConfigFile(const std::string& filename) {
   std::lock_guard<std::mutex> lock(mutex_);
   if (filename.empty()) return;
 

src/configuration.h

@@ -24,7 +24,7 @@ namespace google {
 class MetadataAgentConfiguration {
  public:
   MetadataAgentConfiguration();
-  MetadataAgentConfiguration(const std::string& filename);
+  int ParseArguments(int ac, char** av);
 
   // Shared configuration.
   const std::string& ProjectId() const {
@@ -35,6 +35,10 @@ class MetadataAgentConfiguration {
     std::lock_guard<std::mutex> lock(mutex_);
     return credentials_file_;
   }
+  bool VerboseLogging() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return verbose_logging_;
+  }
   // Metadata API server configuration options.
   int MetadataApiNumThreads() const {
     std::lock_guard<std::mutex> lock(mutex_);
@@ -102,9 +106,12 @@ class MetadataAgentConfiguration {
   }
 
  private:
+  void ParseConfigFile(const std::string& filename);
+
   mutable std::mutex mutex_;
   std::string project_id_;
   std::string credentials_file_;
+  bool verbose_logging_;
   int metadata_api_num_threads_;
   int metadata_api_port_;
   int metadata_reporter_interval_seconds_;

src/docker.cc

@@ -45,7 +45,9 @@ DockerReader::DockerReader(const MetadataAgentConfiguration& config)
 
 std::vector<PollingMetadataUpdater::ResourceMetadata>
     DockerReader::MetadataQuery() const {
-  LOG(INFO) << "Docker Query called";
+  if (config_.VerboseLogging()) {
+    LOG(INFO) << "Docker Query called";
+  }
   const std::string zone = environment_.InstanceZone();
   const std::string docker_endpoint(config_.DockerEndpointHost() +
                                     "v" + config_.DockerApiVersion() +
@@ -60,9 +62,13 @@ std::vector<PollingMetadataUpdater::ResourceMetadata>
   try {
     http::local_client::response list_response = client.get(list_request);
     Timestamp collected_at = std::chrono::system_clock::now();
-    LOG(INFO) << "List response: " << body(list_response);
+    if (config_.VerboseLogging()) {
+      LOG(INFO) << "List response: " << body(list_response);
+    }
     json::value parsed_list = json::Parser::FromString(body(list_response));
-    LOG(INFO) << "Parsed list: " << *parsed_list;
+    if (config_.VerboseLogging()) {
+      LOG(INFO) << "Parsed list: " << *parsed_list;
+    }
     const json::Array* container_list = parsed_list->As<json::Array>();
     for (const json::value& element : *container_list) {
       try {
@@ -71,10 +77,14 @@ std::vector<PollingMetadataUpdater::ResourceMetadata>
         // Inspect the container.
         http::local_client::request inspect_request(docker_endpoint + "/" + id + "/json");
         http::local_client::response inspect_response = client.get(inspect_request);
-        LOG(INFO) << "Inspect response: " << body(inspect_response);
+        if (config_.VerboseLogging()) {
+          LOG(INFO) << "Inspect response: " << body(inspect_response);
+        }
         json::value parsed_metadata =
             json::Parser::FromString(body(inspect_response));
-        LOG(INFO) << "Parsed metadata: " << *parsed_metadata;
+        if (config_.VerboseLogging()) {
+          LOG(INFO) << "Parsed metadata: " << *parsed_metadata;
+        }
         const MonitoredResource resource("docker_container", {
           {"location", zone},
           {"container_id", id},