From f3bfd0b35964e05fbab9ee9d11fcd87844deae86 Mon Sep 17 00:00:00 2001 From: George Hicken Date: Tue, 19 Jun 2018 09:48:06 -0700 Subject: [PATCH 1/5] Correct suite retry units, delete pxe artifacts --- tests/resources/Nimbus-Util.robot | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot index 9564ee29e9..c5b8a00be8 100644 --- a/tests/resources/Nimbus-Util.robot +++ b/tests/resources/Nimbus-Util.robot @@ -222,7 +222,7 @@ Cleanup Nimbus Folders [Arguments] ${deletePXE}=${false} Open Connection %{NIMBUS_GW} Wait Until Keyword Succeeds 2 min 30 sec Login %{NIMBUS_USER} %{NIMBUS_PASSWORD} - Run Keyword If ${deletePXE} Execute Command ${NIMBUS_LOCATION} rm -rf public_html/pxe/* + Run Keyword If ${deletePXE} Execute Command ${NIMBUS_LOCATION} rm -rf public_html/pxe/* public_html/pxeinstall/* Execute Command ${NIMBUS_LOCATION} rm -rf %{BUILD_TAG} Close connection @@ -580,9 +580,9 @@ Get Name of First Local Storage For Host # NIMBUS_RETRY_ATTEMPTS # NIMBUS_RETRY_DELAY Nimbus Suite Setup - [Arguments] ${keyword} ${attempts}=1x ${delay}=1m @{varargs} + [Arguments] ${keyword} ${attempts}=1 ${delay}=1m @{varargs} - ${useAttempts}= Get Environment Variable NIMBUS_RETRY_ATTEMPTS ${attempts}x + ${useAttempts}= Get Environment Variable NIMBUS_RETRY_ATTEMPTS ${attempts} ${useDelay}= Get Environment Variable NIMBUS_RETRY_DELAY ${delay} - Wait Until Keyword Succeeds ${useAttempts} ${useDelay} ${keyword} @{varargs} + Wait Until Keyword Succeeds ${useAttempts}x ${useDelay} ${keyword} @{varargs} From 9f2247272ab2b47264eecca8596b6aa41c1d894a Mon Sep 17 00:00:00 2001 From: George Hicken Date: Tue, 19 Jun 2018 11:41:08 -0700 Subject: [PATCH 2/5] Integration tests fail earlier and sleep less Checks an error return from a nimbus provisioning call instead of trying to infer status solely from console output. Reduce sleep in inner retry loops during nimbus provisioning to 1m instead of 5m. This should really be parameterized or removed completely in favour of the outer retry logic unless there's significant improvement in success probability from retrying at that level. --- tests/nightly/jenkins-nightly-run.sh | 12 +++++++++--- tests/resources/Nimbus-Util.robot | 9 +++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh index f42b7729b6..a5e2d84af3 100755 --- a/tests/nightly/jenkins-nightly-run.sh +++ b/tests/nightly/jenkins-nightly-run.sh @@ -26,9 +26,16 @@ DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs" DEFAULT_VCH_BUILD="*" DEFAULT_TESTCASES=("tests/manual-test-cases/Group5-Functional-Tests" "tests/manual-test-cases/Group13-vMotion" "tests/manual-test-cases/Group21-Registries" "tests/manual-test-cases/Group23-Future-Tests") +DEFAULT_PARALLEL_JOBS=4 +DEFAULT_RUN_AS_OPS_USER=0 + VIC_BINARY_PREFIX="vic_" -export RUN_AS_OPS_USER=0 +# This is exported to propagate into the pybot processes launched by pabot +export RUN_AS_OPS_USER=${RUN_AS_OPS_USER:-${DEFAULT_RUN_AS_OPS_USER}} + +PARALLEL_JOBS=${PARLLEL_JOBS:-${DEFAULT_PARALLEL_JOBS}} + if [[ $1 != "6.0" && $1 != "6.5" && $1 != "6.7" ]]; then echo "Please specify a target version. One of: 6.0, 6.5, 6.7" @@ -95,8 +102,7 @@ else echo "Tarball extraction passed, Running nightlies test.." fi - -pabot --processes 4 --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases[@]}" +pabot --processes ${PARALLEL_JOBS} --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases[@]}" cat ${target}/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log # See if any VMs leaked diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot index c5b8a00be8..e4c4ff4cb7 100644 --- a/tests/resources/Nimbus-Util.robot +++ b/tests/resources/Nimbus-Util.robot @@ -59,8 +59,8 @@ Deploy Nimbus ESXi Server \ ${status}= Run Keyword And Return Status Should Contain ${out} To manage this VM use \ Exit For Loop If ${status} \ Log To Console ${out} - \ Log To Console Nimbus deployment ${IDX} failed, trying again in 5 minutes - \ Sleep 5 minutes + \ Log To Console Nimbus deployment ${IDX} failed, trying again in 1 minutes + \ Sleep 1 minutes # Now grab the IP address and return the name and ip for later use @{out}= Split To Lines ${out} @@ -117,6 +117,7 @@ Deploy Multiple Nimbus ESXi Servers in Parallel \ ${result}= Wait For Process ${pid} \ Log ${result.stdout} \ Log ${result.stderr} + \ Should Be Equal As Integers ${result.rc} 0 &{ips}= Create Dictionary :FOR ${name} IN @{names} @@ -146,8 +147,8 @@ Deploy Nimbus vCenter Server \ # Make sure the deploy actually worked \ ${status}= Run Keyword And Return Status Should Contain ${out} Overall Status: Succeeded \ Exit For Loop If ${status} - \ Log To Console Nimbus deployment ${IDX} failed, trying again in 5 minutes - \ Sleep 5 minutes + \ Log To Console Nimbus deployment ${IDX} failed, trying again in 1 minute + \ Sleep 1 minutes # Now grab the IP address and return the name and ip for later use @{out}= Split To Lines ${out} From 961e67cee7ce5b6666274e431972f5decd400263 Mon Sep 17 00:00:00 2001 From: George Hicken Date: Tue, 19 Jun 2018 17:06:55 -0700 Subject: [PATCH 3/5] Fix compile errors in concurrent test tool --- tests/concurrent/concurrent.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/concurrent/concurrent.go b/tests/concurrent/concurrent.go index f1ff7ee6d2..410b8efabf 100644 --- a/tests/concurrent/concurrent.go +++ b/tests/concurrent/concurrent.go @@ -33,6 +33,7 @@ import ( "github.com/vmware/govmomi/vim25/types" "github.com/vmware/vic/lib/guest" "github.com/vmware/vic/lib/spec" + "github.com/vmware/vic/pkg/trace" "github.com/vmware/vic/pkg/version" "github.com/vmware/vic/pkg/vsphere/datastore" "github.com/vmware/vic/pkg/vsphere/session" @@ -298,10 +299,8 @@ func main() { if config.Start { startFunc := func(i int) error { - _, err := tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.Task, error) { - return vms[i].PowerOn(ctx) - }) - return err + op := trace.FromContext(ctx, "power on") + return vms[i].PowerOn(op) } wrap(startFunc, start) From 9bc6546df8b232b2f90362d4075066c69a8cc72e Mon Sep 17 00:00:00 2001 From: George Hicken Date: Tue, 19 Jun 2018 17:18:25 -0700 Subject: [PATCH 4/5] Correct injecton of retry attempts/delay --- tests/resources/Nimbus-Util.robot | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot index e4c4ff4cb7..e99db1ca30 100644 --- a/tests/resources/Nimbus-Util.robot +++ b/tests/resources/Nimbus-Util.robot @@ -581,9 +581,9 @@ Get Name of First Local Storage For Host # NIMBUS_RETRY_ATTEMPTS # NIMBUS_RETRY_DELAY Nimbus Suite Setup - [Arguments] ${keyword} ${attempts}=1 ${delay}=1m @{varargs} + [Arguments] ${keyword} @{varargs} - ${useAttempts}= Get Environment Variable NIMBUS_RETRY_ATTEMPTS ${attempts} - ${useDelay}= Get Environment Variable NIMBUS_RETRY_DELAY ${delay} + ${useAttempts}= Get Environment Variable NIMBUS_RETRY_ATTEMPTS 1 + ${useDelay}= Get Environment Variable NIMBUS_RETRY_DELAY 1m Wait Until Keyword Succeeds ${useAttempts}x ${useDelay} ${keyword} @{varargs} From f461a4edc95a565904a18ddb8e530be5162e4b99 Mon Sep 17 00:00:00 2001 From: George Hicken Date: Wed, 20 Jun 2018 06:51:41 -0700 Subject: [PATCH 5/5] Retry add hosts to dvs in test setup Previously there were retries of different durations wrapped around the outside of the Add Host To Distributed Switch keyword. If it may need retrying then the operation should do so to avoid this cluster, and those cases where a retry was not present. --- tests/concurrent/concurrent.go | 7 ++++--- .../5-1-Distributed-Switch.robot | 2 +- .../Group5-Functional-Tests/5-2-Cluster.robot | 6 +++--- .../5-4-High-Availability.robot | 2 +- .../Group5-Functional-Tests/5-8-DRS.robot | 2 +- tests/resources/Nimbus-Util.robot | 10 ++++++++-- tests/resources/nimbus-testbeds/vic-vsan.rb | 2 +- 7 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/concurrent/concurrent.go b/tests/concurrent/concurrent.go index 410b8efabf..f1ff7ee6d2 100644 --- a/tests/concurrent/concurrent.go +++ b/tests/concurrent/concurrent.go @@ -33,7 +33,6 @@ import ( "github.com/vmware/govmomi/vim25/types" "github.com/vmware/vic/lib/guest" "github.com/vmware/vic/lib/spec" - "github.com/vmware/vic/pkg/trace" "github.com/vmware/vic/pkg/version" "github.com/vmware/vic/pkg/vsphere/datastore" "github.com/vmware/vic/pkg/vsphere/session" @@ -299,8 +298,10 @@ func main() { if config.Start { startFunc := func(i int) error { - op := trace.FromContext(ctx, "power on") - return vms[i].PowerOn(op) + _, err := tasks.WaitForResult(ctx, func(ctx context.Context) (tasks.Task, error) { + return vms[i].PowerOn(ctx) + }) + return err } wrap(startFunc, start) diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot index 664b5a2099..bdf0a92b19 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot @@ -62,7 +62,7 @@ Distributed Switch Setup :FOR ${IDX} IN RANGE ${esx_number} \ ${out}= Run govc host.add -hostname=@{esx_ips}[${IDX}] -username=root -dc=${datacenter} -password=${NIMBUS_ESX_PASSWORD} -noverify=true \ Should Contain ${out} OK - \ Wait Until Keyword Succeeds 5x 15 seconds Add Host To Distributed Switch @{esx_ips}[${IDX}] + \ Add Host To Distributed Switch @{esx_ips}[${IDX}] Log To Console Deploy VIC to the VC cluster Set Environment Variable TEST_URL_ARRAY ${vc_ip} diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot index f276979570..55e2120690 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot @@ -69,9 +69,9 @@ Cluster Setup Create Three Distributed Port Groups ha-datacenter - Wait Until Keyword Succeeds 5x 1 minute Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[0] - Wait Until Keyword Succeeds 5x 1 minute Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[1] - Wait Until Keyword Succeeds 5x 1 minute Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[2] + Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[0] + Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[1] + Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[2] Log To Console Enable DRS on the cluster ${out}= Run govc cluster.change -drs-enabled /ha-datacenter/host/cls diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot index 18e73e63af..02c52771c5 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot @@ -136,7 +136,7 @@ High Availability Setup \ Should Contain ${out} OK Log To Console Add all the hosts to the distributed switch - Wait Until Keyword Succeeds 5x 5min Add Host To Distributed Switch /${datacenter}/host/cls + Add Host To Distributed Switch /${datacenter}/host/cls Log To Console Enable HA and DRS on the cluster ${out}= Run govc cluster.change -drs-enabled -ha-enabled /${datacenter}/host/cls diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot index c1d7ad9c7d..a303471813 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot @@ -63,7 +63,7 @@ DRS Setup Should Contain ${out} OK Log To Console Add all the hosts to the distributed switch - Wait Until Keyword Succeeds 5x 5min Add Host To Distributed Switch /ha-datacenter/host/cls + Add Host To Distributed Switch /ha-datacenter/host/cls Log To Console Deploy VIC to the VC cluster Set Environment Variable TEST_URL_ARRAY ${vc-ip} diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot index e99db1ca30..2c2de194de 100644 --- a/tests/resources/Nimbus-Util.robot +++ b/tests/resources/Nimbus-Util.robot @@ -188,6 +188,8 @@ Deploy Nimbus vCenter Server Async [Return] ${out} # Deploys a nimbus testbed based on the specified testbed spec and options +# Calls Nimbus Cleanup first as a precaution. Impact on concurrent builds +# is unknown. # user [required] - nimbus user # password [required] - password for nimbus user # args [optional] - args to pass into testbeddeploy @@ -195,6 +197,8 @@ Deploy Nimbus vCenter Server Async Deploy Nimbus Testbed [Arguments] ${user} ${password} ${args}= ${spec}=${EMPTY} + Run Keyword And Ignore Error Cleanup Nimbus Folders deletePxe=${true} + Open Connection %{NIMBUS_GW} Wait Until Keyword Succeeds 2 min 30 sec Login ${user} ${password} @@ -223,6 +227,8 @@ Cleanup Nimbus Folders [Arguments] ${deletePXE}=${false} Open Connection %{NIMBUS_GW} Wait Until Keyword Succeeds 2 min 30 sec Login %{NIMBUS_USER} %{NIMBUS_PASSWORD} + # TODO: this may need pabot shared resource locking around it for multiple jobs. We're likely making use of the + # retry paths currently but it's not good practice. Run Keyword If ${deletePXE} Execute Command ${NIMBUS_LOCATION} rm -rf public_html/pxe/* public_html/pxeinstall/* Execute Command ${NIMBUS_LOCATION} rm -rf %{BUILD_TAG} Close connection @@ -369,7 +375,7 @@ Setup Network For Simple VC Cluster ${out}= Run govc dvs.portgroup.add -nports 12 -dc=${datacenter} -dvs=test-ds bridge Should Contain ${out} OK - Wait Until Keyword Succeeds 10x 3 minutes Add Host To Distributed Switch /${datacenter}/host/${cluster} test-ds + Add Host To Distributed Switch /${datacenter}/host/${cluster} test-ds Log To Console Enable DRS on the cluster ${out}= Run govc cluster.change -drs-enabled /${datacenter}/host/${cluster} @@ -397,7 +403,7 @@ Create Three Distributed Port Groups Add Host To Distributed Switch [Arguments] ${host} ${dvs}=test-ds Log To Console \nAdd host(s) to the distributed switch - ${out}= Run govc dvs.add -dvs=${dvs} -pnic=vmnic1 ${host} + ${out}= Wait Until Keyword Succeeds 10x 30s Run govc dvs.add -dvs=${dvs} -pnic=vmnic1 ${host} Should Contain ${out} OK Disable TLS On ESX Host diff --git a/tests/resources/nimbus-testbeds/vic-vsan.rb b/tests/resources/nimbus-testbeds/vic-vsan.rb index c764e70d99..b747e2f960 100644 --- a/tests/resources/nimbus-testbeds/vic-vsan.rb +++ b/tests/resources/nimbus-testbeds/vic-vsan.rb @@ -22,7 +22,7 @@ { "name" => "esx.#{idx}", "vc" => "vc.0", - "style" => "pxeInstall", + "style" => esxStyle, "desiredPassword" => "e2eFunctionalTest", 'numMem' => 13 * 1024, 'numCPUs' => 4,