From ac4fac8f9b6fb84cb6497ab857982e423f384e5f Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Tue, 15 Dec 2020 18:17:52 +0000 Subject: [PATCH 1/2] traffic_manager should not retry on disk failure --- iocore/cache/Cache.cc | 10 +++++++--- src/traffic_manager/traffic_manager.cc | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 8d2d9e13af0..88c37138ce4 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -729,6 +729,8 @@ CacheProcessor::start_internal(int flags) } Emergency("Cache initialization failed - only %d out of %d disks were valid and all were required.", gndisks, theCacheStore.n_disks_in_config); + } else if (this->waitForCache() == 2 && static_cast(gndisks) < theCacheStore.n_disks_in_config) { + Warning("Cache initialization incomplete - only %d out of %d disks were valid.", gndisks, theCacheStore.n_disks_in_config); } // If we got here, we have enough disks to proceed @@ -784,7 +786,9 @@ CacheProcessor::diskInitialized() if (cb_after_init) { cb_after_init(); } - Fatal("Cache initialization failed - only %d of %d disks were available.", gndisks, theCacheStore.n_disks_in_config); + Emergency("Cache initialization failed - only %d of %d disks were available.", gndisks, theCacheStore.n_disks_in_config); + } else if (this->waitForCache() == 2) { + Warning("Cache initialization incomplete - only %d of %d disks were available.", gndisks, theCacheStore.n_disks_in_config); } } @@ -1052,7 +1056,7 @@ CacheProcessor::cacheInitialized() // TS-3848 if (CACHE_INIT_FAILED == CacheProcessor::initialized && cacheProcessor.waitForCache() > 1) { - Fatal("Cache initialization failed with cache required, exiting."); + Emergency("Cache initialization failed with cache required, exiting."); } } @@ -2062,7 +2066,7 @@ Cache::open_done() // TS-3848 if (ready == CACHE_INIT_FAILED && cacheProcessor.waitForCache() >= 2) { - Fatal("Failed to initialize cache host table"); + Emergency("Failed to initialize cache host table"); } cacheProcessor.cacheInitialized(); diff --git a/src/traffic_manager/traffic_manager.cc b/src/traffic_manager/traffic_manager.cc index 4d507b07e42..329c782f8f5 100644 --- a/src/traffic_manager/traffic_manager.cc +++ b/src/traffic_manager/traffic_manager.cc @@ -492,7 +492,8 @@ main(int argc, const char **argv) char *tsArgs = nullptr; int disable_syslog = false; char userToRunAs[MAX_LOGIN + 1]; - RecInt fds_throttle = -1; + RecInt fds_throttle = -1; + bool printed_unrecoverable = false; ArgumentDescription argument_descriptions[] = { {"proxyOff", '-', "Disable proxy", "F", &proxy_off, nullptr, nullptr}, @@ -831,9 +832,12 @@ main(int argc, const char **argv) } else { just_started++; } - } else { /* Give the proxy a chance to fire up */ - if (!lmgmt->proxy_recoverable) { + } else { + // Even if we shouldn't try the start the proxy again, leave manager around to + // avoid external automated restarts + if (!lmgmt->proxy_recoverable && !printed_unrecoverable) { mgmt_log("[main] Proxy is un-recoverable. Proxy will not be relaunched.\n"); + printed_unrecoverable = true; } just_started++; From d41cc2e76ec0d832bb42c66a7cf3e3d574da0a64 Mon Sep 17 00:00:00 2001 From: Susan Hinrichs Date: Wed, 16 Dec 2020 16:20:46 +0000 Subject: [PATCH 2/2] Fix comment typo --- src/traffic_manager/traffic_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/traffic_manager/traffic_manager.cc b/src/traffic_manager/traffic_manager.cc index 329c782f8f5..4d4ea3406f0 100644 --- a/src/traffic_manager/traffic_manager.cc +++ b/src/traffic_manager/traffic_manager.cc @@ -833,7 +833,7 @@ main(int argc, const char **argv) just_started++; } } else { - // Even if we shouldn't try the start the proxy again, leave manager around to + // Even if we shouldn't try to start the proxy again, leave manager around to // avoid external automated restarts if (!lmgmt->proxy_recoverable && !printed_unrecoverable) { mgmt_log("[main] Proxy is un-recoverable. Proxy will not be relaunched.\n");