From 11e36d5d6105a930c9c281d2c4c719dd49348803 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Tue, 8 May 2018 18:29:14 -0400 Subject: [PATCH 1/7] Shut down gracefully on SIGTERM. --- src/agent.cc | 5 +++++ src/agent.h | 3 +++ src/api_server.cc | 3 ++- src/metadatad.cc | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/agent.cc b/src/agent.cc index 57507b1a..d8003ba1 100644 --- a/src/agent.cc +++ b/src/agent.cc @@ -36,4 +36,9 @@ void MetadataAgent::start() { config_, &store_, config_.MetadataReporterIntervalSeconds())); } +void MetadataAgent::stop() { + metadata_api_server_.reset(); + reporter_.reset(); +} + } diff --git a/src/agent.h b/src/agent.h index 05364671..507a3cdf 100644 --- a/src/agent.h +++ b/src/agent.h @@ -41,6 +41,9 @@ class MetadataAgent { // Starts serving. void start(); + // Stops serving. + void stop(); + const Configuration& config() const { return config_; } diff --git a/src/api_server.cc b/src/api_server.cc index 6768b9a4..92593748 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -84,11 +84,12 @@ MetadataApiServer::MetadataApiServer(const Configuration& config, server_pool_() { for (int i : boost::irange(0, server_threads)) { - server_pool_.emplace_back(&HttpServer::run, &server_); + server_pool_.emplace_back([=]() { server_.run(); }); } } MetadataApiServer::~MetadataApiServer() { + server_.stop(); for (auto& thread : server_pool_) { thread.join(); } diff --git a/src/metadatad.cc b/src/metadatad.cc index bd2f0bb8..f78d34fc 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -14,12 +14,39 @@ * limitations under the License. **/ +#include +#include +#include + #include "agent.h" #include "configuration.h" #include "docker.h" #include "instance.h" #include "kubernetes.h" +namespace google { +namespace { + +struct CleanupState { + CleanupState( + std::initializer_list updaters_, MetadataAgent* server_) + : updaters(updaters_), server(server_) {} + std::vector updaters; + MetadataAgent* server; +}; +const CleanupState* cleanup_state; + +} // namespace +} // google + +extern "C" [[noreturn]] void handle_sigterm(int signum) { + google::cleanup_state->server->stop(); + for (google::MetadataUpdater* updater : google::cleanup_state->updaters) { + updater->stop(); + } + std::exit(128 + signum); +} + int main(int ac, char** av) { google::Configuration config; int parse_result = config.ParseArguments(ac, av); @@ -33,6 +60,11 @@ int main(int ac, char** av) { google::DockerUpdater docker_updater(config, server.mutable_store()); google::KubernetesUpdater kubernetes_updater(config, server.health_checker(), server.mutable_store()); + google::cleanup_state = new google::CleanupState( + {&instance_updater, &docker_updater, &kubernetes_updater}, + &server); + std::signal(SIGTERM, handle_sigterm); + instance_updater.start(); docker_updater.start(); kubernetes_updater.start(); From 6a0a2c54d455d0aa2367bb5f3658da58be6626cd Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Tue, 8 May 2018 19:33:49 -0400 Subject: [PATCH 2/7] Attempt to gracefully shut down watch threads. --- src/kubernetes.cc | 16 ++++++++++++++++ src/kubernetes.h | 1 + src/metadatad.cc | 7 ++++++- src/updater.cc | 10 +++++----- src/updater.h | 4 ++++ 5 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/kubernetes.cc b/src/kubernetes.cc index 8594b4aa..1b1bf088 100644 --- a/src/kubernetes.cc +++ b/src/kubernetes.cc @@ -1321,6 +1321,22 @@ void KubernetesUpdater::StartUpdater() { } } +void KubernetesUpdater::StopUpdater() { + // TODO: How do we interrupt a watch thread? + if (config().KubernetesUseWatch()) { + node_watch_thread_.join(); + pod_watch_thread_.join(); + if (config().KubernetesClusterLevelMetadata() && + config().KubernetesServiceMetadata()) { + service_watch_thread_.join(); + endpoints_watch_thread_.join(); + } + } else { + // Only stop polling if watch is disabled. + PollingMetadataUpdater::StopUpdater(); + } +} + void KubernetesUpdater::MetadataCallback( std::vector&& result_vector) { for (MetadataUpdater::ResourceMetadata& result : result_vector) { diff --git a/src/kubernetes.h b/src/kubernetes.h index 757d49ff..7304b08f 100644 --- a/src/kubernetes.h +++ b/src/kubernetes.h @@ -231,6 +231,7 @@ class KubernetesUpdater : public PollingMetadataUpdater { bool ShouldStartUpdater() const; void StartUpdater(); + void StopUpdater(); private: // Metadata watcher callback. diff --git a/src/metadatad.cc b/src/metadatad.cc index f78d34fc..f9f1a1da 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include "agent.h" #include "configuration.h" @@ -40,10 +41,14 @@ const CleanupState* cleanup_state; } // google extern "C" [[noreturn]] void handle_sigterm(int signum) { - google::cleanup_state->server->stop(); + std::cerr << "Caught SIGTERM; shutting down" << std::endl; + std::cerr << "Stopping updaters" << std::endl; for (google::MetadataUpdater* updater : google::cleanup_state->updaters) { updater->stop(); } + std::cerr << "Stopping server" << std::endl; + google::cleanup_state->server->stop(); + std::cerr << "Exiting" << std::endl; std::exit(128 + signum); } diff --git a/src/updater.cc b/src/updater.cc index 4286df9c..02bd9ee4 100644 --- a/src/updater.cc +++ b/src/updater.cc @@ -77,7 +77,7 @@ bool PollingMetadataUpdater::ShouldStartUpdater() const { void PollingMetadataUpdater::StartUpdater() { timer_.lock(); if (config().VerboseLogging()) { - LOG(INFO) << "Timer locked"; + LOG(INFO) << "Locked timer for " << name(); } reporter_thread_ = std::thread([=]() { PollForMetadata(); }); } @@ -85,7 +85,7 @@ void PollingMetadataUpdater::StartUpdater() { void PollingMetadataUpdater::StopUpdater() { timer_.unlock(); if (config().VerboseLogging()) { - LOG(INFO) << "Timer unlocked"; + LOG(INFO) << "Unlocked timer for " << name(); } } @@ -99,7 +99,7 @@ void PollingMetadataUpdater::PollForMetadata() { } // An unlocked timer means we should stop updating. if (config().VerboseLogging()) { - LOG(INFO) << "Trying to unlock the timer"; + LOG(INFO) << "Trying to unlock the timer for " << name(); } auto start = std::chrono::high_resolution_clock::now(); auto wakeup = start + period_; @@ -113,7 +113,7 @@ void PollingMetadataUpdater::PollForMetadata() { if (config().VerboseLogging()) { LOG(INFO) << " Timer unlock timed out after " << std::chrono::duration_cast(now - start).count() - << "s (good)"; + << "s (good) for " << name(); } start = now; wakeup = start + period_; @@ -121,7 +121,7 @@ void PollingMetadataUpdater::PollForMetadata() { } } while (!done); if (config().VerboseLogging()) { - LOG(INFO) << "Timer unlocked (stop polling)"; + LOG(INFO) << "Timer unlocked (stop polling) for " << name(); } } diff --git a/src/updater.h b/src/updater.h index 4e47e37d..7e7d4b9a 100644 --- a/src/updater.h +++ b/src/updater.h @@ -116,6 +116,10 @@ class MetadataUpdater { store_->UpdateMetadata(result.resource_, std::move(result.metadata_)); } + const std::string& name() const { + return name_; + } + const Configuration& config() const { return config_; } From a580e7adde3e2fe39f415144cd9c11e52266305c Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Fri, 11 May 2018 16:15:36 -0400 Subject: [PATCH 3/7] Wait at the end of main, rather than in the destructor. Stop the server before the updaters. --- src/api_server.cc | 1 + src/kubernetes.cc | 18 ++++++++++++++---- src/metadatad.cc | 8 ++++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/api_server.cc b/src/api_server.cc index 92593748..7794798a 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -90,6 +90,7 @@ MetadataApiServer::MetadataApiServer(const Configuration& config, MetadataApiServer::~MetadataApiServer() { server_.stop(); + LOG(INFO) << "API server stopped"; for (auto& thread : server_pool_) { thread.join(); } diff --git a/src/kubernetes.cc b/src/kubernetes.cc index 1b1bf088..6712991a 100644 --- a/src/kubernetes.cc +++ b/src/kubernetes.cc @@ -1324,13 +1324,23 @@ void KubernetesUpdater::StartUpdater() { void KubernetesUpdater::StopUpdater() { // TODO: How do we interrupt a watch thread? if (config().KubernetesUseWatch()) { - node_watch_thread_.join(); - pod_watch_thread_.join(); +#if 0 + if (node_watch_thread_.joinable()) { + node_watch_thread_.join(); + } + if (pod_watch_thread_.joinable()) { + pod_watch_thread_.join(); + } if (config().KubernetesClusterLevelMetadata() && config().KubernetesServiceMetadata()) { - service_watch_thread_.join(); - endpoints_watch_thread_.join(); + if (service_watch_thread_.joinable()) { + service_watch_thread_.join(); + } + if (endpoints_watch_thread_.joinable()) { + endpoints_watch_thread_.join(); + } } +#endif } else { // Only stop polling if watch is disabled. PollingMetadataUpdater::StopUpdater(); diff --git a/src/metadatad.cc b/src/metadatad.cc index f9f1a1da..4bfd0686 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -42,12 +42,12 @@ const CleanupState* cleanup_state; extern "C" [[noreturn]] void handle_sigterm(int signum) { std::cerr << "Caught SIGTERM; shutting down" << std::endl; + std::cerr << "Stopping server" << std::endl; + google::cleanup_state->server->stop(); std::cerr << "Stopping updaters" << std::endl; for (google::MetadataUpdater* updater : google::cleanup_state->updaters) { updater->stop(); } - std::cerr << "Stopping server" << std::endl; - google::cleanup_state->server->stop(); std::cerr << "Exiting" << std::endl; std::exit(128 + signum); } @@ -59,6 +59,8 @@ int main(int ac, char** av) { return parse_result < 0 ? 0 : parse_result; } + std::mutex server_wait_mutex; + server_wait_mutex.lock(); google::MetadataAgent server(config); google::InstanceUpdater instance_updater(config, server.mutable_store()); @@ -75,4 +77,6 @@ int main(int ac, char** av) { kubernetes_updater.start(); server.start(); + // Wait forever for the server to shut down. + std::lock_guard await_server_shutdown(server_wait_mutex); } From be3ec2c50634a8bfeabe1a52adfe1ab16a21de4b Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Mon, 14 May 2018 15:43:23 -0400 Subject: [PATCH 4/7] Encapsulate waiting and shutdown in the CleanupState class. Also clarify the contract for StopUpdater. --- src/metadatad.cc | 42 +++++++++++++++++++++++++++--------------- src/updater.h | 1 + 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/metadatad.cc b/src/metadatad.cc index 4bfd0686..5d54315d 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -28,12 +28,30 @@ namespace google { namespace { -struct CleanupState { +class CleanupState { + public: CleanupState( - std::initializer_list updaters_, MetadataAgent* server_) - : updaters(updaters_), server(server_) {} - std::vector updaters; - MetadataAgent* server; + std::initializer_list updaters, MetadataAgent* server) + : updaters_(updaters), server_(server) { server_wait_mutex_.lock(); } + + void StopAll() const { + std::cerr << "Stopping server" << std::endl; + server_->stop(); + std::cerr << "Stopping updaters" << std::endl; + for (MetadataUpdater* updater : updaters_) { + updater->stop(); + } + server_wait_mutex_.unlock(); + } + + void Wait() const { + std::lock_guard await_server_shutdown(server_wait_mutex_); + } + + private: + mutable std::mutex server_wait_mutex_; + std::vector updaters_; + MetadataAgent* server_; }; const CleanupState* cleanup_state; @@ -42,12 +60,7 @@ const CleanupState* cleanup_state; extern "C" [[noreturn]] void handle_sigterm(int signum) { std::cerr << "Caught SIGTERM; shutting down" << std::endl; - std::cerr << "Stopping server" << std::endl; - google::cleanup_state->server->stop(); - std::cerr << "Stopping updaters" << std::endl; - for (google::MetadataUpdater* updater : google::cleanup_state->updaters) { - updater->stop(); - } + google::cleanup_state->StopAll(); std::cerr << "Exiting" << std::endl; std::exit(128 + signum); } @@ -59,8 +72,6 @@ int main(int ac, char** av) { return parse_result < 0 ? 0 : parse_result; } - std::mutex server_wait_mutex; - server_wait_mutex.lock(); google::MetadataAgent server(config); google::InstanceUpdater instance_updater(config, server.mutable_store()); @@ -77,6 +88,7 @@ int main(int ac, char** av) { kubernetes_updater.start(); server.start(); - // Wait forever for the server to shut down. - std::lock_guard await_server_shutdown(server_wait_mutex); + + // Wait for the server to shut down. + google::cleanup_state->Wait(); } diff --git a/src/updater.h b/src/updater.h index 7e7d4b9a..99504b13 100644 --- a/src/updater.h +++ b/src/updater.h @@ -104,6 +104,7 @@ class MetadataUpdater { virtual void StartUpdater() = 0; // Internal method for stopping the updater's logic. + // This method should not perform any blocking operations (e.g., wait). virtual void StopUpdater() = 0; // Updates the resource map in the store. From 7071958762682362f4720ccea231e20cdec62d2f Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Tue, 15 May 2018 12:32:56 -0400 Subject: [PATCH 5/7] Address feedback. - Function renames (mostly Stop->NotifyStop). - Remove dead code. --- src/agent.cc | 4 ++-- src/agent.h | 4 ++-- src/kubernetes.cc | 23 +++-------------------- src/kubernetes.h | 2 +- src/metadatad.cc | 16 ++++++++-------- src/updater.cc | 8 ++++---- src/updater.h | 12 ++++++------ test/updater_unittest.cc | 10 +++++----- 8 files changed, 31 insertions(+), 48 deletions(-) diff --git a/src/agent.cc b/src/agent.cc index d8003ba1..e2b4ae44 100644 --- a/src/agent.cc +++ b/src/agent.cc @@ -28,7 +28,7 @@ MetadataAgent::MetadataAgent(const Configuration& config) MetadataAgent::~MetadataAgent() {} -void MetadataAgent::start() { +void MetadataAgent::Start() { metadata_api_server_.reset(new MetadataApiServer( config_, store_, config_.MetadataApiNumThreads(), "0.0.0.0", config_.MetadataApiPort())); @@ -36,7 +36,7 @@ void MetadataAgent::start() { config_, &store_, config_.MetadataReporterIntervalSeconds())); } -void MetadataAgent::stop() { +void MetadataAgent::Stop() { metadata_api_server_.reset(); reporter_.reset(); } diff --git a/src/agent.h b/src/agent.h index 507a3cdf..8f4669e5 100644 --- a/src/agent.h +++ b/src/agent.h @@ -39,10 +39,10 @@ class MetadataAgent { ~MetadataAgent(); // Starts serving. - void start(); + void Start(); // Stops serving. - void stop(); + void Stop(); const Configuration& config() const { return config_; diff --git a/src/kubernetes.cc b/src/kubernetes.cc index 6712991a..a04d4e51 100644 --- a/src/kubernetes.cc +++ b/src/kubernetes.cc @@ -1321,29 +1321,12 @@ void KubernetesUpdater::StartUpdater() { } } -void KubernetesUpdater::StopUpdater() { - // TODO: How do we interrupt a watch thread? +void KubernetesUpdater::NotifyStopUpdater() { if (config().KubernetesUseWatch()) { -#if 0 - if (node_watch_thread_.joinable()) { - node_watch_thread_.join(); - } - if (pod_watch_thread_.joinable()) { - pod_watch_thread_.join(); - } - if (config().KubernetesClusterLevelMetadata() && - config().KubernetesServiceMetadata()) { - if (service_watch_thread_.joinable()) { - service_watch_thread_.join(); - } - if (endpoints_watch_thread_.joinable()) { - endpoints_watch_thread_.join(); - } - } -#endif + // TODO: How do we interrupt a watch thread? } else { // Only stop polling if watch is disabled. - PollingMetadataUpdater::StopUpdater(); + PollingMetadataUpdater::NotifyStopUpdater(); } } diff --git a/src/kubernetes.h b/src/kubernetes.h index 7304b08f..2e3d50be 100644 --- a/src/kubernetes.h +++ b/src/kubernetes.h @@ -231,7 +231,7 @@ class KubernetesUpdater : public PollingMetadataUpdater { bool ShouldStartUpdater() const; void StartUpdater(); - void StopUpdater(); + void NotifyStopUpdater(); private: // Metadata watcher callback. diff --git a/src/metadatad.cc b/src/metadatad.cc index 5d54315d..9fe0244e 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -34,12 +34,12 @@ class CleanupState { std::initializer_list updaters, MetadataAgent* server) : updaters_(updaters), server_(server) { server_wait_mutex_.lock(); } - void StopAll() const { + void StartShutdown() const { std::cerr << "Stopping server" << std::endl; - server_->stop(); + server_->Stop(); std::cerr << "Stopping updaters" << std::endl; for (MetadataUpdater* updater : updaters_) { - updater->stop(); + updater->NotifyStop(); } server_wait_mutex_.unlock(); } @@ -60,7 +60,7 @@ const CleanupState* cleanup_state; extern "C" [[noreturn]] void handle_sigterm(int signum) { std::cerr << "Caught SIGTERM; shutting down" << std::endl; - google::cleanup_state->StopAll(); + google::cleanup_state->StartShutdown(); std::cerr << "Exiting" << std::endl; std::exit(128 + signum); } @@ -83,11 +83,11 @@ int main(int ac, char** av) { &server); std::signal(SIGTERM, handle_sigterm); - instance_updater.start(); - docker_updater.start(); - kubernetes_updater.start(); + instance_updater.Start(); + docker_updater.Start(); + kubernetes_updater.Start(); - server.start(); + server.Start(); // Wait for the server to shut down. google::cleanup_state->Wait(); diff --git a/src/updater.cc b/src/updater.cc index 02bd9ee4..6c3d9d41 100644 --- a/src/updater.cc +++ b/src/updater.cc @@ -30,7 +30,7 @@ MetadataUpdater::MetadataUpdater(const Configuration& config, MetadataUpdater::~MetadataUpdater() {} -void MetadataUpdater::start() throw(ConfigurationValidationError) { +void MetadataUpdater::Start() throw(ConfigurationValidationError) { ValidateStaticConfiguration(); if (ShouldStartUpdater()) { @@ -41,8 +41,8 @@ void MetadataUpdater::start() throw(ConfigurationValidationError) { } } -void MetadataUpdater::stop() { - StopUpdater(); +void MetadataUpdater::NotifyStop() { + NotifyStopUpdater(); } PollingMetadataUpdater::PollingMetadataUpdater( @@ -82,7 +82,7 @@ void PollingMetadataUpdater::StartUpdater() { reporter_thread_ = std::thread([=]() { PollForMetadata(); }); } -void PollingMetadataUpdater::StopUpdater() { +void PollingMetadataUpdater::NotifyStopUpdater() { timer_.unlock(); if (config().VerboseLogging()) { LOG(INFO) << "Unlocked timer for " << name(); diff --git a/src/updater.h b/src/updater.h index 99504b13..3d21f86a 100644 --- a/src/updater.h +++ b/src/updater.h @@ -70,10 +70,10 @@ class MetadataUpdater { virtual ~MetadataUpdater(); // Starts updating. - void start() throw(ConfigurationValidationError); + void Start() throw(ConfigurationValidationError); - // Stops updating. - void stop(); + // Notifies the updater to stop updating. + void NotifyStop(); using UpdateCallback = std::function&&)>; @@ -103,9 +103,9 @@ class MetadataUpdater { // Internal method for starting the updater's logic. virtual void StartUpdater() = 0; - // Internal method for stopping the updater's logic. + // Internal method for notifying the updater's to stop its logic. // This method should not perform any blocking operations (e.g., wait). - virtual void StopUpdater() = 0; + virtual void NotifyStopUpdater() = 0; // Updates the resource map in the store. void UpdateResourceCallback(const ResourceMetadata& result) { @@ -151,7 +151,7 @@ class PollingMetadataUpdater : public MetadataUpdater { using MetadataUpdater::ValidateDynamicConfiguration; bool ShouldStartUpdater() const; void StartUpdater(); - void StopUpdater(); + void NotifyStopUpdater(); private: friend class InstanceTest; diff --git a/test/updater_unittest.cc b/test/updater_unittest.cc index be37d904..2e729325 100644 --- a/test/updater_unittest.cc +++ b/test/updater_unittest.cc @@ -109,7 +109,7 @@ class MockMetadataUpdater : public MetadataUpdater { void StartUpdater() { call_sequence_.push_back("StartUpdater"); } - void StopUpdater() {} + void NotifyStopUpdater() {} mutable std::vector call_sequence_; @@ -125,7 +125,7 @@ TEST_F(ValidationOrderingTest, FailedStaticCheckStopsOtherChecks) { /*fail_static=*/true, /*should_start=*/true, /*fail_dynamic=*/true); - EXPECT_THROW(updater.start(), MetadataUpdater::ConfigurationValidationError); + EXPECT_THROW(updater.Start(), MetadataUpdater::ConfigurationValidationError); EXPECT_EQ( std::vector({ "ValidateStaticConfiguration", @@ -139,7 +139,7 @@ TEST_F(ValidationOrderingTest, FalseShouldStartUpdaterStopsDynamicChecks) { /*fail_static=*/false, /*should_start=*/false, /*fail_dynamic=*/false); - EXPECT_NO_THROW(updater.start()); + EXPECT_NO_THROW(updater.Start()); EXPECT_EQ( std::vector({ "ValidateStaticConfiguration", @@ -154,7 +154,7 @@ TEST_F(ValidationOrderingTest, FailedDynamicCheckStopsStartUpdater) { /*fail_static=*/false, /*should_start=*/true, /*fail_dynamic=*/true); - EXPECT_THROW(updater.start(), MetadataUpdater::ConfigurationValidationError); + EXPECT_THROW(updater.Start(), MetadataUpdater::ConfigurationValidationError); EXPECT_EQ( std::vector({ "ValidateStaticConfiguration", @@ -170,7 +170,7 @@ TEST_F(ValidationOrderingTest, AllChecksPassedInvokesStartUpdater) { /*fail_static=*/false, /*should_start=*/true, /*fail_dynamic=*/false); - EXPECT_NO_THROW(updater.start()); + EXPECT_NO_THROW(updater.Start()); EXPECT_EQ( std::vector({ "ValidateStaticConfiguration", From 9d430603b155b79f84438d1731fc44b17afe71a3 Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Tue, 15 May 2018 13:11:11 -0400 Subject: [PATCH 6/7] Make agent Stop non-blocking. --- src/agent.cc | 4 ++-- src/api_server.cc | 6 +++++- src/api_server.h | 3 +++ src/metadatad.cc | 3 +++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/agent.cc b/src/agent.cc index e2b4ae44..77f9717b 100644 --- a/src/agent.cc +++ b/src/agent.cc @@ -37,8 +37,8 @@ void MetadataAgent::Start() { } void MetadataAgent::Stop() { - metadata_api_server_.reset(); - reporter_.reset(); + metadata_api_server_->Stop(); + // TODO: Notify the metadata reporter as well. } } diff --git a/src/api_server.cc b/src/api_server.cc index 7794798a..84036935 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -88,9 +88,13 @@ MetadataApiServer::MetadataApiServer(const Configuration& config, } } -MetadataApiServer::~MetadataApiServer() { +void MetadataApiServer::Stop() { server_.stop(); LOG(INFO) << "API server stopped"; +} + +MetadataApiServer::~MetadataApiServer() { + Stop(); for (auto& thread : server_pool_) { thread.join(); } diff --git a/src/api_server.h b/src/api_server.h index 850eb8be..b700a27b 100644 --- a/src/api_server.h +++ b/src/api_server.h @@ -43,6 +43,9 @@ class MetadataApiServer { int server_threads, const std::string& host, int port); ~MetadataApiServer(); + // Stops the server and closes the listening socket. + void Stop(); + private: friend class ApiServerTest; diff --git a/src/metadatad.cc b/src/metadatad.cc index 9fe0244e..fef25250 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -24,6 +24,7 @@ #include "docker.h" #include "instance.h" #include "kubernetes.h" +#include "time.h" namespace google { namespace { @@ -42,6 +43,8 @@ class CleanupState { updater->NotifyStop(); } server_wait_mutex_.unlock(); + // Give the notifications some time to propagate. + std::this_thread::sleep_for(time::seconds(0.1)); } void Wait() const { From 2ffea567b224e72f97679887d615da11e436171b Mon Sep 17 00:00:00 2001 From: Igor Peshansky Date: Thu, 24 May 2018 16:02:28 -0400 Subject: [PATCH 7/7] Report success when exiting via SIGTERM. --- src/metadatad.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metadatad.cc b/src/metadatad.cc index fef25250..7363d2f5 100644 --- a/src/metadatad.cc +++ b/src/metadatad.cc @@ -65,7 +65,7 @@ extern "C" [[noreturn]] void handle_sigterm(int signum) { std::cerr << "Caught SIGTERM; shutting down" << std::endl; google::cleanup_state->StartShutdown(); std::cerr << "Exiting" << std::endl; - std::exit(128 + signum); + std::exit(0); // SIGTERM means graceful shutdown, so report success. } int main(int ac, char** av) {