From 3a8728067da5394ec9fb3e479d845abfa64bcdcb Mon Sep 17 00:00:00 2001 From: George Hicken Date: Thu, 14 Jun 2018 10:26:34 -0700 Subject: [PATCH] Parameterize nightly job script This parameterizes the nightly job runner script so that we can pass in the configuration elements that were previously hard coded. The primary difference is it's possible to run a user specified subset of the test cases. Also removes references in the test definitions to specific user directories. There is a mostly duplicated version of the job runner files called jenkins-drs-disabled-run.sh - as best I can determine these files are not unified only because some of the nightly tests have not been confirmed to work when the test executor is in a container. --- .../5-11-Multiple-Cluster.robot | 3 +- .../5-15-NFS-Datastore.robot | 2 +- .../5-22-NFS-Volume.robot | 2 +- .../5-26-Static-IP-Address.robot | 1 - .../5-27-Selenium-Grid.robot | 3 +- .../5-6-1-VSAN-Simple.robot | 2 - .../5-6-2-VSAN-Complex.robot | 2 - tests/nightly/jenkins-nightly-run.sh | 140 ++++++++++-------- tests/nightly/upload-logs.sh | 53 +++---- tests/resources/Nimbus-Util.robot | 46 +++--- 10 files changed, 135 insertions(+), 119 deletions(-) diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot index 6eeebc80c2..24a90676c4 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot @@ -24,7 +24,7 @@ Multiple Cluster Setup [Timeout] 110 minutes Run Keyword And Ignore Error Nimbus Cleanup Single VM '*5-11-multiple-cluster*' ${false} Log To Console \nStarting testbed deploy... - ${out}= Deploy Nimbus Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-multiple-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-multiple-cluster.rb --runName 5-11-multiple-cluster + ${out}= Deploy Nimbus Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} spec=vic-multiple-cluster.rb args=--noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-multiple-cluster --runName 5-11-multiple-cluster Log ${out} Open Connection %{NIMBUS_GW} @@ -56,7 +56,6 @@ Multiple Cluster Setup *** Test Cases *** Test Log To Console \nStarting test... - Custom Testbed Keepalive /dbc/pa-dbc1111/mhagen Install VIC Appliance To Test Server certs=${false} vol=default Run Regression Tests diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot index 10e3fa8414..3008e5f590 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot @@ -23,7 +23,7 @@ NFS Datastore Setup [Timeout] 110 minutes Run Keyword And Ignore Error Nimbus Cleanup Single VM '*5-15-nfs-datastore*' ${false} Log To Console \nStarting testbed deploy... - ${out}= Deploy Nimbus Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-simple-cluster.rb --runName 5-15-nfs-datastore + ${out}= Deploy Nimbus Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} spec=vic-simple-cluster.rb args=--noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster --runName 5-15-nfs-datastore Log ${out} Open Connection %{NIMBUS_GW} diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot index 5311f66209..0590553911 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot @@ -38,7 +38,7 @@ Setup ESX And NFS Suite Run Keyword And Ignore Error Nimbus Cleanup ${list} ${false} Log To Console \nStarting test... - ${nfs} ${nfs_ro} ${esx1} ${nfs_ip} ${nfs_ro_ip} ${esx1_ip}= Deploy Simple NFS Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} additional-args=--testbedSpecRubyFile /dbc/w3-dbc302/rashok/vic-nfs.rb --esxBuild ${ESX_VERSION} --esxPxeDir ${ESX_VERSION} --plugin testng + ${nfs} ${nfs_ro} ${esx1} ${nfs_ip} ${nfs_ro_ip} ${esx1_ip}= Deploy Simple NFS Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} spec=vic-nfs.rb args=--esxBuild ${ESX_VERSION} --esxPxeDir ${ESX_VERSION} --plugin testng Set Suite Variable @{list} ${esx1} ${nfs} ${nfs_ro} Set Suite Variable ${ESX1} ${esx1} diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot index 65adb78c3f..937569d72f 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot @@ -27,7 +27,6 @@ Setup VC With Static IP *** Test Cases *** Test Log To Console \nStarting test... - Custom Testbed Keepalive /dbc/pa-dbc1111/mhagen Install VIC Appliance To Test Server additional-args=--public-network-ip &{static}[ip]/&{static}[netmask] --public-network-gateway &{static}[gateway] --dns-server 10.170.16.48 Run Regression Tests \ No newline at end of file diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot index ac0fef639b..573baef0e7 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot @@ -24,7 +24,7 @@ Selenium Grid Test Setup Log To Console Starting testbed deployment... Run Keyword And Ignore Error Nimbus Cleanup ${list} ${false} ${name}= Evaluate 'vic-5-27-' + str(random.randint(1000,9999)) modules=random - ${out}= Deploy Nimbus Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-iscsi-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-iscsi-cluster.rb --runName ${name} + ${out}= Deploy Nimbus Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} spec=vic-iscsi-cluster.rb args=--noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-iscsi-cluster --runName ${name} Log ${out} Open Connection %{NIMBUS_GW} @@ -71,7 +71,6 @@ Wait Until Selenium Node Is Ready *** Test Cases *** Test Log To Console Starting Selenium Grid test... - Custom Testbed Keepalive /dbc/pa-dbc1111/mhagen Install VIC Appliance To Test Server diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot index 0efe749b44..abb4e6bd85 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot @@ -68,8 +68,6 @@ Check VSAN DOMs In Datastore Simple VSAN Wait Until Keyword Succeeds 10x 30s Check VSAN DOMs In Datastore %{TEST_DATASTORE} - Custom Testbed Keepalive /dbc/pa-dbc1111/mhagen - Install VIC Appliance To Test Server Run Regression Tests Cleanup VIC Appliance On Test Server diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot index 5197a7eefe..5c0306b646 100644 --- a/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot +++ b/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot @@ -71,8 +71,6 @@ Check VSAN DOMs In Datastore Complex VSAN Wait Until Keyword Succeeds 10x 30s Check VSAN DOMs In Datastore %{TEST_DATASTORE} - Custom Testbed Keepalive /dbc/pa-dbc1111/mhagen - Install VIC Appliance To Test Server Run Regression Tests Cleanup VIC Appliance On Test Server diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh index cc38897ec5..f0be5f58c0 100755 --- a/tests/nightly/jenkins-nightly-run.sh +++ b/tests/nightly/jenkins-nightly-run.sh @@ -16,84 +16,102 @@ ESX_60_VERSION="ob-5251623" VC_60_VERSION="ob-5112509" +ESX_65_VERSION="ob-7867845" +VC_65_VERSION="ob-7867539" + ESX_67_VERSION="ob-8169922" VC_67_VERSION="ob-8217866" +DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs" +DEFAULT_VCH_BUILD="*" +DEFAULT_TESTCASES="tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests" + + export RUN_AS_OPS_USER=0 if [[ $1 != "6.0" && $1 != "6.5" && $1 != "6.7" ]]; then - echo "Please specify a target cluster. One of: 6.0, 6.5, 6.7" + echo "Please specify a target version. One of: 6.0, 6.5, 6.7" exit 1 fi + +# process the CLI arguments target="$1" -echo "Target cluster: "$target - -input=$(gsutil ls -l gs://vic-engine-builds/vic_* | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | cut -d '/' -f 4) -buildNumber=${input:4} - -n=0 - until [ $n -ge 5 ] - do - echo "Retry.. $n" - echo "Downloading gcp file $input" - wget https://storage.googleapis.com/vic-engine-builds/$input - if [ -f "$input" ] - then - echo "File found.." - break - else - echo "File NOT found" - fi - n=$[$n+1] - sleep 15 - done - -n=0 - until [ $n -ge 5 ] - do - mkdir bin - echo "Extracting .tar.gz" - tar xvzf $input -C bin/ --strip 1 - if [ -f "bin/vic-machine-linux" ] - then - echo "tar extraction complete.." - canContinue="Yes" - break - else - echo "tar extraction failed" - canContinue="No" - rm -rf bin - fi - n=$[$n+1] - sleep 15 - done - -if [[ $canContinue = "No" ]]; then +echo "Target version: ${target}" +shift +# Take the remaining CLI arguments as a test case list +testcases="${*:-$DEFAULT_TESTCASES}" + +# TODO: the version downloaded by this logic is not coupled with the tests that will be run against it. This should be altered to pull a version that matches the commit SHA of the tests +# we will be running or similar mechanism. +VCH_BUILD=${VCH_BUILD:-${DEFAULT_VCH_BUILD}} +input=$(gsutil ls -l gs://vic-engine-builds/vic_${VCH_BUILD} | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | cut -d '/' -f 4) +VCH_BUILD=${input:4} + +# Enforce short SHA +GIT_COMMIT=${GIT_COMMIT:0:7} + +case "$target" in + "6.0") + excludes="--exclude nsx" + ESX_BUILD=${ESX_BUILD:-$ESX_60_VERSION} + VC_BUILD=${VC_BUILD:-$VC_60_VERSION} + ;; + "6.5") + ESX_BUILD=${ESX_BUILD:-$ESX_65_VERSION} + VC_BUILD=${VC_BUILD:-$VC_65_VERSION} + ;; + "6.7") + excludes="--exclude nsx --exclude hetero" + ESX_BUILD=${ESX_BUILD:-$ESX_67_VERSION} + VC_BUILD=${VC_BUILD:-$VC_67_VERSION} + ;; +esac + +LOG_UPLOAD_DEST="${LOG_UPLOAD_DEST:-${DEFAULT_LOG_UPLOAD_DEST}}" + +n=0 && rm -f "${input}" +until [ $n -ge 5 -o -f "${input}" ]; do + echo "Retry.. $n" + echo "Downloading gcp file ${input}" + wget -nv https://storage.googleapis.com/vic-engine-builds/${input} + + ((n++)) + sleep 15 +done + +echo "Extracting .tar.gz" +mkdir bin && tar xvzf ${input} -C bin/ --strip 1 + +if [ ! -f "bin/vic-machine-linux" ]; then echo "Tarball extraction failed..quitting the run" - break + rm -rf bin + exit else + VCH_COMMIT=$(bin/vic-machine-linux version | awk -F '-' '{print $NF}') echo "Tarball extraction passed, Running nightlies test.." fi -if [[ $target == "6.0" ]]; then - echo "Executing nightly tests on vSphere 6.0" - pabot --processes 4 --removekeywords TAG:secret --exclude nsx --variable ESX_VERSION:$ESX_60_VERSION --variable VC_VERSION:$VC_60_VERSION -d 60/$i tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests - cat 60/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log -elif [[ $target == "6.5" ]]; then - echo "Executing nightly tests on vSphere 6.5" - pabot --processes 4 --removekeywords TAG:secret -d 65/$i tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests - cat 65/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log -elif [[ $target == "6.7" ]]; then - echo "Executing nightly tests on vSphere 6.7" - pabot --processes 4 --removekeywords TAG:secret --exclude nsx --exclude hetero --variable ESX_VERSION:$ESX_67_VERSION --variable VC_VERSION:$VC_67_VERSION -d 67/$i tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests - cat 67/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log + +pabot --processes 4 --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases}" +cat ${target}/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log + +# See if any VMs leaked +# TODO: should be a warning until clean, then changed to a failure if any leak +echo "There should not be any VMs listed here" +echo "=======================================" +timeout 60s sshpass -p ${NIMBUS_PASSWORD} ssh -o StrictHostKeyChecking\=no ${NIMBUS_USER}@${NIMBUS_GW} nimbus-ctl list +echo "=======================================" +echo "If VMs are listed we should investigate why they are leaking" + +# archive the logs +logarchive="logs_vch-${VCH_BUILD}-${VCH_COMMIT}_test-${BUILD_ID}-${GIT_COMMIT}_${BUILD_TIMESTAMP}.zip" +/usr/bin/zip -9 -r "${logarchive}" "${target}" *.zip *.log *.debug *.tgz +if [ $? -eq 0 ]; then + tests/nightly/upload-logs.sh ${logarchive} ${LOG_UPLOAD_DEST} fi + # Pretty up the email results sed -i -e 's/^/
/g' console.log sed -i -e 's|PASS|PASS|g' console.log sed -i -e 's|FAIL|FAIL|g' console.log -# See if any VMs leaked -timeout 60s sshpass -p $NIMBUS_PASSWORD ssh -o StrictHostKeyChecking\=no $NIMBUS_USER@$NIMBUS_GW nimbus-ctl list - -tests/nightly/upload-logs.sh $target_$BUILD_TIMESTAMP diff --git a/tests/nightly/upload-logs.sh b/tests/nightly/upload-logs.sh index 8811c5b69c..82e72afdf3 100755 --- a/tests/nightly/upload-logs.sh +++ b/tests/nightly/upload-logs.sh @@ -14,28 +14,23 @@ # limitations under the License. # -echo "Upload logs" +source=${1:?A source file must be specified} +# The target destination in cloud storage. If the destination ends with a / it will be treated as a directory. +# If not it _may_ be treated as a directory or a file depending on current remote objects. see gsutil cp doc. +dest=${2:?A bucket must be specified, eg vic-ci-logs, or vic-ci-logs/user/branch/} -set -x -gsutil version -l -set +x - -outfile="vic_nightly_logs_"$1".zip" -echo $outfile +if [ ! -r "${source}" ]; then + echo "Specified source file does not exist or cannot be read: ${source}" + exit 1 +fi -if [ -d "60" ]; then - /usr/bin/zip -9 -r $outfile 60 *.zip *.log *.debug *.tgz -elif [ -d "65" ]; then - /usr/bin/zip -9 -r $outfile 65 *.zip *.log *.debug *.tgz -elif [ -d "67" ]; then - /usr/bin/zip -9 -r $outfile 67 *.zip *.log *.debug *.tgz -else - echo "No output directories to upload!" - exit 1 +if [ ${dest:0:1} == "/" ]; then + echo "Destination must start with a bucket name and no leading /" + exit 1 fi # GC credentials -keyfile=~/vic-ci-logs.key +keyfile=~/${dest%%/*}.key botofile=~/.boto if [ ! -f $keyfile ]; then echo -en $GS_PRIVATE_KEY > $keyfile @@ -50,16 +45,22 @@ if [ ! -f $botofile ]; then echo "default_project_id = $GS_PROJECT_ID" >> $botofile fi -if [ -f "$outfile" ]; then - gsutil cp $outfile gs://vic-ci-logs - echo "----------------------------------------------" +echo "----------------------------------------------" +echo "Uploading logs to ${dest}" +echo "----------------------------------------------" +if gsutil cp "${source}" "gs://${dest}"; then + url="https://console.cloud.google.com/m/cloudstorage/b/${dest}/o/${source}?authuser=1" + echo "$url" > log-download.url echo "Download test logs here:" - echo "https://console.cloud.google.com/m/cloudstorage/b/vic-ci-logs/o/$outfile?authuser=1" - echo "----------------------------------------------" + echo "$url" else - echo "No log output file to upload" + echo "Log upload faled. Dumping gsutil version logic" + set -x + gsutil version -l + set +x fi +echo "----------------------------------------------" + +rm -f $keyfile + -if [ -f "$keyfile" ]; then - rm -f $keyfile -fi diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot index 0ffe35ad08..25138317fd 100644 --- a/tests/resources/Nimbus-Util.robot +++ b/tests/resources/Nimbus-Util.robot @@ -45,12 +45,6 @@ Fetch POD ${pod}= Fetch From Left ${out} : [return] ${pod} -Custom Testbed Keepalive - [Tags] secret - [Arguments] ${folder} - ${out}= Run Secret SSHPASS command %{NIMBUS_USER} '%{NIMBUS_PASSWORD}' touch ${folder} - [Return] ${out} - Deploy Nimbus ESXi Server [Arguments] ${user} ${password} ${version}=${ESX_VERSION} ${tls_disabled}=True ${name}= Evaluate 'ESX-' + str(random.randint(1000,9999)) + str(time.clock()) modules=random,time @@ -192,13 +186,22 @@ Deploy Nimbus vCenter Server Async ${out}= Run Secret SSHPASS command %{NIMBUS_USER} '%{NIMBUS_PASSWORD}' '${NIMBUS_LOCATION} nimbus-vcvadeploy --lease=0.25 --vcvaBuild ${version} ${name}' [Return] ${out} +# Deploys a nimbus testbed based on the specified testbed spec and options +# user [required] - nimbus user +# password [required] - password for nimbus user +# spec [optional] - name of spec file in tests/resources/nimbus-testbeds +# args [optiona;] - args to pass into testbeddeploy Deploy Nimbus Testbed - [Arguments] ${user} ${password} ${testbed} + [Arguments] ${user} ${password} ${spec}= ${args}= + Open Connection %{NIMBUS_GW} Wait Until Keyword Succeeds 2 min 30 sec Login ${user} ${password} + Run Keyword Unless ${spec} == ${EMPTY} Set Test Variable ${specarg} --testbedSpecRubyFile ./%{BUILD_TAG}/testbeds/${spec} + Run Keyword Unless ${spec} == ${EMPTY} Put File tests/resources/nimbus-testbeds/${spec} destination=./%{BUILD_TAG}/testbeds/ + :FOR ${IDX} IN RANGE 1 5 - \ ${out}= Execute Command ${NIMBUS_LOCATION} nimbus-testbeddeploy --lease 0.25 ${testbed} + \ ${out}= Execute Command ${NIMBUS_LOCATION} nimbus-testbeddeploy --lease 0.25 ${specarg} ${args} \ Log ${out} \ # Make sure the deploy actually worked \ ${status}= Run Keyword And Return Status Should Contain ${out} "deployment_result"=>"PASS" @@ -215,26 +218,27 @@ Kill Nimbus Server Log ${out} Close connection -Cleanup Nimbus PXE folder - [Arguments] ${user} ${password} +Cleanup Nimbus Folders + [Arguments] ${deletePXE}=${false} Open Connection %{NIMBUS_GW} - Wait Until Keyword Succeeds 2 min 30 sec Login ${user} ${password} - ${out}= Execute Command ${NIMBUS_LOCATION} rm -rf public_html/pxe/* + Wait Until Keyword Succeeds 2 min 30 sec Login %{NIMBUS_USER} %{NIMBUS_PASSWORD} + Run Keyword If ${deletePXE} Execute Command ${NIMBUS_LOCATION} rm -rf public_html/pxe/* + Execute Command ${NIMBUS_LOCATION} rm -rf %{BUILD_TAG} Close connection +# Cleans up a list of VMs and deletes the pxe folder on nimbus gateway Nimbus Cleanup [Arguments] ${vm_list} ${collect_log}=True ${dontDelete}=${false} - Run Keyword If ${collect_log} Run Keyword And Continue On Failure Gather Logs From Test Server - Run Keyword And Ignore Error Cleanup Nimbus PXE folder %{NIMBUS_USER} %{NIMBUS_PASSWORD} - Return From Keyword If ${dontDelete} ${list}= Catenate @{vm_list} - Run Keyword And Ignore Error Kill Nimbus Server %{NIMBUS_USER} %{NIMBUS_PASSWORD} ${list} + Run Keyword Nimbus Cleanup Single VM ${list} ${collect_log} ${dontDelete} ${true} +# Cleans up a vm (or space separated string list of vms) but does not delete pxe folder on nimbus gateway Nimbus Cleanup Single VM - [Arguments] ${vm} ${collect_log}=True ${dontDelete}=${false} + [Arguments] ${vms} ${collect_log}=True ${dontDelete}=${false} ${deletePXE}=${false} Run Keyword If ${collect_log} Run Keyword And Continue On Failure Gather Logs From Test Server + Run Keyword And Ignore Error Cleanup Nimbus Folders ${deletePXE} Return From Keyword If ${dontDelete} - Run Keyword And Ignore Error Kill Nimbus Server %{NIMBUS_USER} %{NIMBUS_PASSWORD} ${vm} + Run Keyword And Ignore Error Kill Nimbus Server %{NIMBUS_USER} %{NIMBUS_PASSWORD} ${vms} Gather Host IPs ${out}= Run govc ls host/cls @@ -409,13 +413,13 @@ Get Vsphere Version \ Run Keyword And Return If ${status} Fetch From Right ${line} ${SPACE} Deploy Simple NFS Testbed - [Arguments] ${user} ${password} ${additional-args}= + [Arguments] ${user} ${password} ${spec}= ${args}= ${name}= Evaluate 'NFS-' + str(random.randint(1000,9999)) + str(time.clock()) modules=random,time Log To Console \nDeploying Nimbus NFS testbed: ${name} Open Connection %{NIMBUS_GW} Wait Until Keyword Succeeds 2 min 30 sec Login ${user} ${password} - ${out}= Execute Command ${NIMBUS_LOCATION} nimbus-testbeddeploy --testbedName nfs --runName ${name} ${additional-args} + ${out}= Execute Command ${NIMBUS_LOCATION} nimbus-testbeddeploy spec=${spec} args=--testbedName nfs --runName ${name} ${args} Log ${out} # Make sure the deploy actually worked Should Contain ${out} ${name}.nfs.0' is up. IP: @@ -488,7 +492,7 @@ Create Simple VC Cluster With Static IP Set Suite Variable ${NIMBUS_LOCATION} NIMBUS_LOCATION=wdc Run Keyword And Ignore Error Nimbus Cleanup ${list} ${false} Log To Console Create a new simple vc cluster with static ip support... - ${out}= Deploy Nimbus Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster-with-static --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-simple-cluster-with-static.rb --runName ${name} + ${out}= Deploy Nimbus Testbed %{NIMBUS_USER} %{NIMBUS_PASSWORD} spec=vic-simple-cluster-with-static.rb args=--noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster-with-static --runName ${name} Log ${out} Open Connection %{NIMBUS_GW}