diff --git a/src/Makefile b/src/Makefile index 0fec29a0..cd510478 100644 --- a/src/Makefile +++ b/src/Makefile @@ -7,7 +7,7 @@ SUBMODULE_DIRS=$(CPP_NETLIB_DIR) $(YAML_CPP_DIR) SED_I=/usr/bin/env sed -i CMAKE=cmake -CXXFLAGS=-std=c++11 -g -DENABLE_KUBERNETES_METADATA -I$(CPP_NETLIB_DIR) -I$(YAML_CPP_DIR)/include +CXXFLAGS=-std=c++11 -g -DENABLE_DOCKER_METADATA -DENABLE_KUBERNETES_METADATA -I$(CPP_NETLIB_DIR) -I$(YAML_CPP_DIR)/include LDFLAGS=-L$(CPP_NETLIB_LIBDIR) -L$(YAML_CPP_LIBDIR) LDLIBS=-lcppnetlib-uri -lcppnetlib-client-connections -lboost_program_options \ -lboost_system -lboost_thread -lpthread -lyajl -lssl -lcrypto -lyaml-cpp diff --git a/src/docker.cc b/src/docker.cc index 18361026..c87d1698 100644 --- a/src/docker.cc +++ b/src/docker.cc @@ -17,9 +17,11 @@ #include "docker.h" #include "local_stream_http.h" +#include #include #include +#include "instance.h" #include "json.h" #include "logging.h" #include "resource.h" @@ -32,16 +34,91 @@ namespace google { namespace { #if 0 -constexpr const char docker_endpoint_host[] = "unix://%2Fvar%2Frun%2Fdocker.sock/"; -constexpr const char docker_api_version[] = "1.23"; +constexpr const char kDockerEndpointHost[] = "unix://%2Fvar%2Frun%2Fdocker.sock/"; +constexpr const char kDockerApiVersion[] = "1.23"; #endif -constexpr const char docker_endpoint_path[] = "/containers"; +constexpr const char kDockerEndpointPath[] = "/containers"; +constexpr const char kDockerContainerResourcePrefix[] = "container"; } DockerReader::DockerReader(const MetadataAgentConfiguration& config) : config_(config), environment_(config) {} +MetadataUpdater::ResourceMetadata DockerReader::GetContainerMetadata( + const json::Object* container, Timestamp collected_at) const + throw(json::Exception) { + const std::string zone = environment_.InstanceZone(); + + const std::string id = container->Get("Id"); + // Inspect the container. + try { + json::value raw_container = QueryDocker( + std::string(kDockerEndpointPath) + "/" + id + "/json"); + if (config_.VerboseLogging()) { + LOG(INFO) << "Parsed metadata: " << *raw_container; + } + + const json::Object* container_desc = raw_container->As(); + const std::string name = container_desc->Get("Name"); + + const std::string created_str = + container_desc->Get("Created"); + Timestamp created_at = rfc3339::FromString(created_str); + + const json::Object* state = container_desc->Get("State"); + bool is_deleted = state->Get("Dead"); + + const MonitoredResource resource("docker_container", { + {"location", zone}, + {"container_id", id}, + }); + + json::value instance_resource = + InstanceReader::InstanceResource(environment_).ToJSON(); + + json::value raw_metadata = json::object({ + {"blobs", json::object({ + {"association", json::object({ + {"version", json::string(config_.MetadataIngestionRawContentVersion())}, + {"raw", json::object({ + {"infrastructureResource", std::move(instance_resource)}, + })}, + })}, + {"api", json::object({ + {"version", json::string(config_.DockerApiVersion())}, + {"raw", std::move(raw_container)}, + })}, + })}, + }); + if (config_.VerboseLogging()) { + LOG(INFO) << "Raw docker metadata: " << *raw_metadata; + } + + const std::string resource_id = boost::algorithm::join( + std::vector{kDockerContainerResourcePrefix, id}, + config_.MetadataApiResourceTypeSeparator()); + const std::string resource_name = boost::algorithm::join( + // The container name reported by Docker will always have a leading '/'. + std::vector{kDockerContainerResourcePrefix, name.substr(1)}, + config_.MetadataApiResourceTypeSeparator()); + return MetadataUpdater::ResourceMetadata( + std::vector{resource_id, resource_name}, + resource, +#ifdef ENABLE_DOCKER_METADATA + MetadataAgent::Metadata(config_.MetadataIngestionRawContentVersion(), + is_deleted, created_at, collected_at, + std::move(raw_metadata)) +#else + MetadataAgent::Metadata::IGNORED() +#endif + ); + } catch (const QueryException& e) { + throw json::Exception("Container " + id + + " disappeared before we could inspect it"); + } +} + std::vector DockerReader::MetadataQuery() const { if (config_.VerboseLogging()) { @@ -50,66 +127,23 @@ std::vector const std::string zone = environment_.InstanceZone(); const std::string docker_endpoint(config_.DockerEndpointHost() + "v" + config_.DockerApiVersion() + - docker_endpoint_path); + kDockerEndpointPath); const std::string container_filter( config_.DockerContainerFilter().empty() ? "" : "&" + config_.DockerContainerFilter()); - http::local_client client; - http::local_client::request list_request( - docker_endpoint + "/json?all=true" + container_filter); std::vector result; try { - http::local_client::response list_response = client.get(list_request); + json::value parsed_list = QueryDocker( + std::string(kDockerEndpointPath) + "/json?all=true" + container_filter); Timestamp collected_at = std::chrono::system_clock::now(); - if (config_.VerboseLogging()) { - LOG(INFO) << "List response: " << body(list_response); - } - json::value parsed_list = json::Parser::FromString(body(list_response)); if (config_.VerboseLogging()) { LOG(INFO) << "Parsed list: " << *parsed_list; } const json::Array* container_list = parsed_list->As(); - for (const json::value& element : *container_list) { + for (const json::value& raw_container : *container_list) { try { - const json::Object* container = element->As(); - const std::string id = container->Get("Id"); - // Inspect the container. - http::local_client::request inspect_request(docker_endpoint + "/" + id + "/json"); - http::local_client::response inspect_response = client.get(inspect_request); - if (config_.VerboseLogging()) { - LOG(INFO) << "Inspect response: " << body(inspect_response); - } - json::value parsed_metadata = - json::Parser::FromString(body(inspect_response)); - if (config_.VerboseLogging()) { - LOG(INFO) << "Parsed metadata: " << *parsed_metadata; - } - - const MonitoredResource resource("docker_container", { - {"location", zone}, - {"container_id", id}, - }); - - const json::Object* container_desc = parsed_metadata->As(); - const std::string name = container_desc->Get("Name"); - - const std::string created_str = - container_desc->Get("Created"); - Timestamp created_at = rfc3339::FromString(created_str); - - const json::Object* state = container_desc->Get("State"); - bool is_deleted = state->Get("Dead"); - - const std::string resource_id = - std::string("container") + config_.MetadataApiResourceTypeSeparator() + id; - // The container name reported by Docker will always have a leading '/'. - const std::string resource_name = - std::string("container") + config_.MetadataApiResourceTypeSeparator() + name.substr(1); - result.emplace_back(std::vector{resource_id, resource_name}, - resource, - MetadataAgent::Metadata(config_.DockerApiVersion(), - is_deleted, created_at, collected_at, - std::move(parsed_metadata))); + const json::Object* container = raw_container->As(); + result.emplace_back(GetContainerMetadata(container, collected_at)); } catch (const json::Exception& e) { LOG(ERROR) << e.what(); continue; @@ -117,10 +151,32 @@ std::vector } } catch (const json::Exception& e) { LOG(ERROR) << e.what(); - } catch (const boost::system::system_error& e) { - LOG(ERROR) << "Failed to communicate with " << docker_endpoint << ": " << e.what(); + } catch (const QueryException& e) { + // Already logged. } return result; } +json::value DockerReader::QueryDocker(const std::string& path) const + throw(QueryException, json::Exception) { + const std::string endpoint(config_.DockerEndpointHost() + + "v" + config_.DockerApiVersion() + + path); + http::local_client client; + http::local_client::request request(endpoint); + if (config_.VerboseLogging()) { + LOG(INFO) << "QueryDocker: Contacting " << endpoint; + } + try { + http::local_client::response response = client.get(request); +#ifdef VERBOSE + LOG(DEBUG) << "QueryDocker: Response: " << body(response); +#endif + return json::Parser::FromString(body(response)); + } catch (const boost::system::system_error& e) { + LOG(ERROR) << "Failed to query " << endpoint << ": " << e.what(); + throw QueryException(endpoint + " -> " + e.what()); + } +} + } diff --git a/src/docker.h b/src/docker.h index 917ee85e..da54446b 100644 --- a/src/docker.h +++ b/src/docker.h @@ -33,6 +33,25 @@ class DockerReader { std::vector MetadataQuery() const; private: + // A representation of all query-related errors. + class QueryException { + public: + QueryException(const std::string& what) : explanation_(what) {} + const std::string& what() const { return explanation_; } + private: + std::string explanation_; + }; + + // Issues a Docker API query at a given path and returns a parsed + // JSON response. The path has to start with "/". + json::value QueryDocker(const std::string& path) const + throw(QueryException, json::Exception); + + // Given a container object, return the associated metadata. + MetadataUpdater::ResourceMetadata GetContainerMetadata( + const json::Object* container, Timestamp collected_at) const + throw(json::Exception); + const MetadataAgentConfiguration& config_; Environment environment_; };