diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot index 664b5a2099..bdf0a92b19 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot @@ -62,7 +62,7 @@ Distributed Switch Setup :FOR ${IDX} IN RANGE ${esx_number} \ ${out}= Run govc host.add -hostname=@{esx_ips}[${IDX}] -username=root -dc=${datacenter} -password=${NIMBUS_ESX_PASSWORD} -noverify=true \ Should Contain ${out} OK - \ Wait Until Keyword Succeeds 5x 15 seconds Add Host To Distributed Switch @{esx_ips}[${IDX}] + \ Add Host To Distributed Switch @{esx_ips}[${IDX}] Log To Console Deploy VIC to the VC cluster Set Environment Variable TEST_URL_ARRAY ${vc_ip} diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot index f276979570..55e2120690 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot @@ -69,9 +69,9 @@ Cluster Setup Create Three Distributed Port Groups ha-datacenter - Wait Until Keyword Succeeds 5x 1 minute Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[0] - Wait Until Keyword Succeeds 5x 1 minute Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[1] - Wait Until Keyword Succeeds 5x 1 minute Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[2] + Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[0] + Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[1] + Add Host To Distributed Switch /ha-datacenter/host/cls/@{esx_ips}[2] Log To Console Enable DRS on the cluster ${out}= Run govc cluster.change -drs-enabled /ha-datacenter/host/cls diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot index 18e73e63af..02c52771c5 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot @@ -136,7 +136,7 @@ High Availability Setup \ Should Contain ${out} OK Log To Console Add all the hosts to the distributed switch - Wait Until Keyword Succeeds 5x 5min Add Host To Distributed Switch /${datacenter}/host/cls + Add Host To Distributed Switch /${datacenter}/host/cls Log To Console Enable HA and DRS on the cluster ${out}= Run govc cluster.change -drs-enabled -ha-enabled /${datacenter}/host/cls diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot index c1d7ad9c7d..a303471813 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot @@ -63,7 +63,7 @@ DRS Setup Should Contain ${out} OK Log To Console Add all the hosts to the distributed switch - Wait Until Keyword Succeeds 5x 5min Add Host To Distributed Switch /ha-datacenter/host/cls + Add Host To Distributed Switch /ha-datacenter/host/cls Log To Console Deploy VIC to the VC cluster Set Environment Variable TEST_URL_ARRAY ${vc-ip} diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh index f42b7729b6..a5e2d84af3 100755 --- a/tests/nightly/jenkins-nightly-run.sh +++ b/tests/nightly/jenkins-nightly-run.sh @@ -26,9 +26,16 @@ DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs" DEFAULT_VCH_BUILD="*" DEFAULT_TESTCASES=("tests/manual-test-cases/Group5-Functional-Tests" "tests/manual-test-cases/Group13-vMotion" "tests/manual-test-cases/Group21-Registries" "tests/manual-test-cases/Group23-Future-Tests") +DEFAULT_PARALLEL_JOBS=4 +DEFAULT_RUN_AS_OPS_USER=0 + VIC_BINARY_PREFIX="vic_" -export RUN_AS_OPS_USER=0 +# This is exported to propagate into the pybot processes launched by pabot +export RUN_AS_OPS_USER=${RUN_AS_OPS_USER:-${DEFAULT_RUN_AS_OPS_USER}} + +PARALLEL_JOBS=${PARLLEL_JOBS:-${DEFAULT_PARALLEL_JOBS}} + if [[ $1 != "6.0" && $1 != "6.5" && $1 != "6.7" ]]; then echo "Please specify a target version. One of: 6.0, 6.5, 6.7" @@ -95,8 +102,7 @@ else echo "Tarball extraction passed, Running nightlies test.." fi - -pabot --processes 4 --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases[@]}" +pabot --processes ${PARALLEL_JOBS} --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases[@]}" cat ${target}/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log # See if any VMs leaked diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot index 9564ee29e9..2c2de194de 100644 --- a/tests/resources/Nimbus-Util.robot +++ b/tests/resources/Nimbus-Util.robot @@ -59,8 +59,8 @@ Deploy Nimbus ESXi Server \ ${status}= Run Keyword And Return Status Should Contain ${out} To manage this VM use \ Exit For Loop If ${status} \ Log To Console ${out} - \ Log To Console Nimbus deployment ${IDX} failed, trying again in 5 minutes - \ Sleep 5 minutes + \ Log To Console Nimbus deployment ${IDX} failed, trying again in 1 minutes + \ Sleep 1 minutes # Now grab the IP address and return the name and ip for later use @{out}= Split To Lines ${out} @@ -117,6 +117,7 @@ Deploy Multiple Nimbus ESXi Servers in Parallel \ ${result}= Wait For Process ${pid} \ Log ${result.stdout} \ Log ${result.stderr} + \ Should Be Equal As Integers ${result.rc} 0 &{ips}= Create Dictionary :FOR ${name} IN @{names} @@ -146,8 +147,8 @@ Deploy Nimbus vCenter Server \ # Make sure the deploy actually worked \ ${status}= Run Keyword And Return Status Should Contain ${out} Overall Status: Succeeded \ Exit For Loop If ${status} - \ Log To Console Nimbus deployment ${IDX} failed, trying again in 5 minutes - \ Sleep 5 minutes + \ Log To Console Nimbus deployment ${IDX} failed, trying again in 1 minute + \ Sleep 1 minutes # Now grab the IP address and return the name and ip for later use @{out}= Split To Lines ${out} @@ -187,6 +188,8 @@ Deploy Nimbus vCenter Server Async [Return] ${out} # Deploys a nimbus testbed based on the specified testbed spec and options +# Calls Nimbus Cleanup first as a precaution. Impact on concurrent builds +# is unknown. # user [required] - nimbus user # password [required] - password for nimbus user # args [optional] - args to pass into testbeddeploy @@ -194,6 +197,8 @@ Deploy Nimbus vCenter Server Async Deploy Nimbus Testbed [Arguments] ${user} ${password} ${args}= ${spec}=${EMPTY} + Run Keyword And Ignore Error Cleanup Nimbus Folders deletePxe=${true} + Open Connection %{NIMBUS_GW} Wait Until Keyword Succeeds 2 min 30 sec Login ${user} ${password} @@ -222,7 +227,9 @@ Cleanup Nimbus Folders [Arguments] ${deletePXE}=${false} Open Connection %{NIMBUS_GW} Wait Until Keyword Succeeds 2 min 30 sec Login %{NIMBUS_USER} %{NIMBUS_PASSWORD} - Run Keyword If ${deletePXE} Execute Command ${NIMBUS_LOCATION} rm -rf public_html/pxe/* + # TODO: this may need pabot shared resource locking around it for multiple jobs. We're likely making use of the + # retry paths currently but it's not good practice. + Run Keyword If ${deletePXE} Execute Command ${NIMBUS_LOCATION} rm -rf public_html/pxe/* public_html/pxeinstall/* Execute Command ${NIMBUS_LOCATION} rm -rf %{BUILD_TAG} Close connection @@ -368,7 +375,7 @@ Setup Network For Simple VC Cluster ${out}= Run govc dvs.portgroup.add -nports 12 -dc=${datacenter} -dvs=test-ds bridge Should Contain ${out} OK - Wait Until Keyword Succeeds 10x 3 minutes Add Host To Distributed Switch /${datacenter}/host/${cluster} test-ds + Add Host To Distributed Switch /${datacenter}/host/${cluster} test-ds Log To Console Enable DRS on the cluster ${out}= Run govc cluster.change -drs-enabled /${datacenter}/host/${cluster} @@ -396,7 +403,7 @@ Create Three Distributed Port Groups Add Host To Distributed Switch [Arguments] ${host} ${dvs}=test-ds Log To Console \nAdd host(s) to the distributed switch - ${out}= Run govc dvs.add -dvs=${dvs} -pnic=vmnic1 ${host} + ${out}= Wait Until Keyword Succeeds 10x 30s Run govc dvs.add -dvs=${dvs} -pnic=vmnic1 ${host} Should Contain ${out} OK Disable TLS On ESX Host @@ -580,9 +587,9 @@ Get Name of First Local Storage For Host # NIMBUS_RETRY_ATTEMPTS # NIMBUS_RETRY_DELAY Nimbus Suite Setup - [Arguments] ${keyword} ${attempts}=1x ${delay}=1m @{varargs} + [Arguments] ${keyword} @{varargs} - ${useAttempts}= Get Environment Variable NIMBUS_RETRY_ATTEMPTS ${attempts}x - ${useDelay}= Get Environment Variable NIMBUS_RETRY_DELAY ${delay} + ${useAttempts}= Get Environment Variable NIMBUS_RETRY_ATTEMPTS 1 + ${useDelay}= Get Environment Variable NIMBUS_RETRY_DELAY 1m - Wait Until Keyword Succeeds ${useAttempts} ${useDelay} ${keyword} @{varargs} + Wait Until Keyword Succeeds ${useAttempts}x ${useDelay} ${keyword} @{varargs} diff --git a/tests/resources/nimbus-testbeds/vic-vsan.rb b/tests/resources/nimbus-testbeds/vic-vsan.rb index c764e70d99..b747e2f960 100644 --- a/tests/resources/nimbus-testbeds/vic-vsan.rb +++ b/tests/resources/nimbus-testbeds/vic-vsan.rb @@ -22,7 +22,7 @@ { "name" => "esx.#{idx}", "vc" => "vc.0", - "style" => "pxeInstall", + "style" => esxStyle, "desiredPassword" => "e2eFunctionalTest", 'numMem' => 13 * 1024, 'numCPUs' => 4,