From f0f169f152dc10f33d63f7004e1a717a34c2f53f Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Thu, 2 Feb 2023 14:16:04 -0600 Subject: [PATCH 1/2] Fix autest flakiness due better Ready Conditions (#9372) A number of tests were flakey because of race conditions between the AuTest framework finishing the Default TestRun process and ending the test, and therefore the traffic server process, before traffic server had time to exercise the expected functionality of the test. This improves the Ready conditions for these tests which should improve their reliability. (cherry picked from commit c175aa45c29124ab01af8f1efa1222fca8ff41c8) --- tests/gold_tests/shutdown/emergency.test.py | 4 +++- tests/gold_tests/shutdown/fatal.test.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/gold_tests/shutdown/emergency.test.py b/tests/gold_tests/shutdown/emergency.test.py index 3e4d9abdef4..e4008f5b19d 100644 --- a/tests/gold_tests/shutdown/emergency.test.py +++ b/tests/gold_tests/shutdown/emergency.test.py @@ -45,7 +45,9 @@ tr.Processes.Default.Command = 'printf "Emergency Shutdown Test"' tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) +tr.Timeout = 5 + ts.ReturnCode = 33 -ts.Ready = 0 # Need this to be 0 because we are testing shutdown, this is to make autest not think ats went away for a bad reason. +ts.Ready = When.FileContains(ts.Disk.traffic_out.Name, "testing emergency shutdown") ts.Disk.traffic_out.Content = Testers.ExcludesExpression('failed to shutdown', 'should NOT contain "failed to shutdown"') ts.Disk.diags_log.Content = Testers.IncludesExpression('testing emergency shutdown', 'should contain "testing emergency shutdown"') diff --git a/tests/gold_tests/shutdown/fatal.test.py b/tests/gold_tests/shutdown/fatal.test.py index a36ea44e515..c2be94c31bd 100644 --- a/tests/gold_tests/shutdown/fatal.test.py +++ b/tests/gold_tests/shutdown/fatal.test.py @@ -45,7 +45,9 @@ tr.Processes.Default.Command = 'printf "Fatal Shutdown Test"' tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(ts) +tr.Timeout = 5 + ts.ReturnCode = 70 -ts.Ready = 0 # Need this to be 0 because we are testing shutdown, this is to make autest not think ats went away for a bad reason. +ts.Ready = When.FileContains(ts.Disk.traffic_out.Name, "testing fatal shutdown") ts.Disk.traffic_out.Content = Testers.ExcludesExpression('failed to shutdown', 'should NOT contain "failed to shutdown"') ts.Disk.diags_log.Content = Testers.IncludesExpression('testing fatal shutdown', 'should contain "testing fatal shutdown"') From 88d387bafad5bbee8a350d3dda6dd690c7ae0006 Mon Sep 17 00:00:00 2001 From: Brian Neradt Date: Wed, 8 Feb 2023 16:01:06 -0600 Subject: [PATCH 2/2] Make autest ts shutdown tests more reliable (#9391) This is a second iteration to further improve some of the autests that exercise ATS shutting down during test execution. See #9372 for the previous patch. The previous update improved reliability, but still had a race condition between autest recognizing ATS process shutdown and ATS writing the expected error log. If it detected the ts process ending before the log was written, the autest would fail because the framework thought the process ended before the Ready condition was satisfied. This patch addresses this by using a separate process to wait upon the log entry instead of making the ts Ready condition the content of the log file. (cherry picked from commit 086501cb58d7b872e955b49de952b554ba7b5685) --- tests/gold_tests/shutdown/emergency.test.py | 17 +++++++++++++---- tests/gold_tests/shutdown/fatal.test.py | 17 +++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/gold_tests/shutdown/emergency.test.py b/tests/gold_tests/shutdown/emergency.test.py index e4008f5b19d..fcc18e85d9e 100644 --- a/tests/gold_tests/shutdown/emergency.test.py +++ b/tests/gold_tests/shutdown/emergency.test.py @@ -40,14 +40,23 @@ # Load plugin Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'emergency_shutdown.so'), ts) -# www.example.com Host tr = Test.AddTestRun() + +# We have to wait upon TS to emit the expected log message, but it cannot be +# the ts Ready criteria because autest might detect the process going away +# before it detects the log message. So we add a separate process that waits +# upon the log message. +watcher = Test.Processes.Process("watcher") +watcher.Command = "sleep 1" +watcher.Ready = When.FileContains(ts.Disk.diags_log.Name, "testing emergency shutdown") +watcher.StartBefore(ts) + tr.Processes.Default.Command = 'printf "Emergency Shutdown Test"' tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.StartBefore(ts) -tr.Timeout = 5 +tr.Processes.Default.StartBefore(watcher) +tr.Timeout = 5 ts.ReturnCode = 33 -ts.Ready = When.FileContains(ts.Disk.traffic_out.Name, "testing emergency shutdown") +ts.Ready = 0 ts.Disk.traffic_out.Content = Testers.ExcludesExpression('failed to shutdown', 'should NOT contain "failed to shutdown"') ts.Disk.diags_log.Content = Testers.IncludesExpression('testing emergency shutdown', 'should contain "testing emergency shutdown"') diff --git a/tests/gold_tests/shutdown/fatal.test.py b/tests/gold_tests/shutdown/fatal.test.py index c2be94c31bd..aa8e8c3303c 100644 --- a/tests/gold_tests/shutdown/fatal.test.py +++ b/tests/gold_tests/shutdown/fatal.test.py @@ -40,14 +40,23 @@ # Load plugin Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsTestPluginsDir, 'fatal_shutdown.so'), ts) -# www.example.com Host tr = Test.AddTestRun() + +# We have to wait upon TS to emit the expected log message, but it cannot be +# the ts Ready criteria because autest might detect the process going away +# before it detects the log message. So we add a separate process that waits +# upon the log message. +watcher = Test.Processes.Process("watcher") +watcher.Command = "sleep 1" +watcher.Ready = When.FileContains(ts.Disk.diags_log.Name, "testing fatal shutdown") +watcher.StartBefore(ts) + tr.Processes.Default.Command = 'printf "Fatal Shutdown Test"' tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.StartBefore(ts) -tr.Timeout = 5 +tr.Processes.Default.StartBefore(watcher) +tr.Timeout = 5 ts.ReturnCode = 70 -ts.Ready = When.FileContains(ts.Disk.traffic_out.Name, "testing fatal shutdown") +ts.Ready = 0 ts.Disk.traffic_out.Content = Testers.ExcludesExpression('failed to shutdown', 'should NOT contain "failed to shutdown"') ts.Disk.diags_log.Content = Testers.IncludesExpression('testing fatal shutdown', 'should contain "testing fatal shutdown"')