From 7d8eedd5916c4f77f1443ef2e63fadea03fe3319 Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Tue, 17 Apr 2018 11:11:35 -0500
Subject: [PATCH 01/31] Update DRS test to our new non-drs support (#7765)
 (#8151)

(cherry picked from commit 453200e7942c395c29400a8dc59e7ebe515d55d7)
---
 .../Group5-Functional-Tests/5-8-DRS.robot                  | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot
index e0df7ea14e..a6690e6e5c 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot
@@ -79,14 +79,13 @@ DRS Setup
 *** Test Cases ***
 Test
     Log To Console  \nStarting test...
-    ${status}  ${message}=  Run Keyword And Ignore Error  Install VIC Appliance To Test Server  certs=${false}  vol=default
-    Should Contain  ${message}  DRS must be enabled to use VIC
-    Should Be Equal As Strings  ${status}  FAIL
+    Install VIC Appliance To Test Server  certs=${false}  vol=default
+    Run Regression Tests
+    Cleanup VIC Appliance On Test Server
 
     Log To Console  Enable DRS on the cluster
     ${out}=  Run  govc cluster.change -drs-enabled /ha-datacenter/host/cls
     Should Be Empty  ${out}
 
     Install VIC Appliance To Test Server  certs=${false}  vol=default
-
     Run Regression Tests

From e13795c56735cecc86a58199420214967add192c Mon Sep 17 00:00:00 2001
From: rogeliosanchez <rogelios@vmware.com>
Date: Thu, 19 Apr 2018 17:33:53 -0500
Subject: [PATCH 02/31] Add retry on vSAN DOM objects in DS check (#7792)
 (#8151)

* Add retry on vSAN DOM objects in DS check

(cherry picked from commit f1c9787f7a03a79612c2ff8a3e16ed4513551f79)
---
 .../5-6-1-VSAN-Simple.robot                     | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot
index 34395af611..0efe749b44 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot
@@ -1,4 +1,4 @@
-# Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+# Copyright 2016-2018 VMware, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -34,7 +34,7 @@ Simple VSAN Setup
     \   Exit For Loop If  ${status}
 
     Set Suite Variable  @{list}  ${user}-${name}.vcva-${VC_VERSION}  ${user}-${name}.esx.0  ${user}-${name}.esx.1  ${user}-${name}.esx.2  ${user}-${name}.esx.3  ${user}-${name}.nfs.0  ${user}-${name}.iscsi.0
-
+    
     Log To Console  Set environment variables up for GOVC
     Set Environment Variable  GOVC_URL  ${vc-ip}
     Set Environment Variable  GOVC_USERNAME  Administrator@vsphere.local
@@ -57,10 +57,16 @@ Simple VSAN Setup
     Set Environment Variable  TEST_RESOURCE  cls
     Set Environment Variable  TEST_TIMEOUT  15m
 
+
+Check VSAN DOMs In Datastore
+    [Arguments]  ${test_datastore}
+    ${out}=  Run  govc datastore.vsan.dom.ls -ds ${test_datastore} -l -o
+    Should Be Empty  ${out}
+
+
 *** Test Cases ***
 Simple VSAN
-    ${out}=  Run  govc datastore.vsan.dom.ls -ds %{TEST_DATASTORE} -l -o
-    Should Be Empty  ${out}
+    Wait Until Keyword Succeeds  10x  30s  Check VSAN DOMs In Datastore  %{TEST_DATASTORE}
 
     Custom Testbed Keepalive  /dbc/pa-dbc1111/mhagen
 
@@ -68,5 +74,4 @@ Simple VSAN
     Run Regression Tests
     Cleanup VIC Appliance On Test Server
 
-    ${out}=  Run  govc datastore.vsan.dom.ls -ds %{TEST_DATASTORE} -l -o
-    Should Be Empty  ${out}
+    Wait Until Keyword Succeeds  10x  30s  Check VSAN DOMs In Datastore  %{TEST_DATASTORE}

From 87f782ec7b46cc8ac88ee371468025b9b0a8594c Mon Sep 17 00:00:00 2001
From: rogeliosanchez <rogelios@vmware.com>
Date: Fri, 20 Apr 2018 13:25:29 -0500
Subject: [PATCH 03/31] Add vSAN DOM check retry for vSAN Complex test (#7799)
 (#8151)

* Add vSAN DOM check retry for vSAN complex

(cherry picked from commit 760046bf1d2d820d3594e2fe3064710aa59e13d5)
---
 .../5-6-2-VSAN-Complex.robot                      | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot
index c6d1604455..5197a7eefe 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot
@@ -60,18 +60,21 @@ VSAN Complex Setup
     Set Environment Variable  TEST_RESOURCE  cluster-vsan-1
     Set Environment Variable  TEST_TIMEOUT  30m
 
-*** Test Cases ***
-Complex VSAN
-    ${out}=  Run  govc datastore.vsan.dom.ls -ds %{TEST_DATASTORE} -l -o
+
+Check VSAN DOMs In Datastore
+    [Arguments]  ${test_datastore}
+    ${out}=  Run  govc datastore.vsan.dom.ls -ds ${test_datastore} -l -o
     Log  ${out}
     Should Be Empty  ${out}
 
+*** Test Cases ***
+Complex VSAN
+    Wait Until Keyword Succeeds  10x  30s  Check VSAN DOMs In Datastore  %{TEST_DATASTORE}
+
     Custom Testbed Keepalive  /dbc/pa-dbc1111/mhagen
 
     Install VIC Appliance To Test Server
     Run Regression Tests
     Cleanup VIC Appliance On Test Server
 
-    ${out}=  Run  govc datastore.vsan.dom.ls -ds %{TEST_DATASTORE} -l -o
-    Log  ${out}
-    Should Be Empty  ${out}
+    Wait Until Keyword Succeeds  10x  30s  Check VSAN DOMs In Datastore  %{TEST_DATASTORE}

From c7ff9b35cc4d19ecb74200125d31935c20fa3a44 Mon Sep 17 00:00:00 2001
From: Jason Morris <10388115+morris-jason@users.noreply.github.com>
Date: Tue, 24 Apr 2018 13:37:00 -0500
Subject: [PATCH 04/31] Add nightly for vicadmin tests on isolated vch (#7793)
 (#8151)

Adds a nightly test that verifies vicadmin functions properly without a
wan connection. Fixes #7102.

(cherry picked from commit d5135e4bf67d7789cf887a2a8c0ad48f1c4aa461)
---
 .../5-26-Static-IP-Address.robot              | 41 +-------
 .../5-28-VICAdmin-Isolated.md                 | 50 ++++++++++
 .../5-28-VICAdmin-Isolated.robot              | 99 +++++++++++++++++++
 .../Group5-Functional-Tests/TestCases.md      |  2 +-
 tests/resources/Nimbus-Util.robot             | 39 ++++++++
 5 files changed, 193 insertions(+), 38 deletions(-)
 create mode 100644 tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.md
 create mode 100644 tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.robot

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot
index ae3571fb13..65adb78c3f 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot
@@ -15,48 +15,15 @@
 *** Settings ***
 Documentation  Test 5-26 - Static IP Address
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Static IP Address Create
+Suite Setup  Setup VC With Static IP
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
+
 *** Keywords ***
-Static IP Address Create
-    [Timeout]    110 minutes
-    Log To Console  Starting Static IP Address test...
-    Set Suite Variable  ${NIMBUS_LOCATION}  NIMBUS_LOCATION=wdc
-    Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
+Setup VC With Static IP
     ${name}=  Evaluate  'vic-5-26-' + str(random.randint(1000,9999))  modules=random
-    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-simple-cluster.rb --runName ${name}
-
-    Open Connection  %{NIMBUS_GW}
-    Wait Until Keyword Succeeds  10 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
-    ${vc-ip}=  Get IP  ${name}.vc.0
-    ${pod}=  Fetch POD  ${name}.vc.0
-    Set Suite Variable  ${NIMBUS_POD}  ${pod}
-    Close Connection
-
-    Set Suite Variable  @{list}  %{NIMBUS_USER}-${name}.esx.0  %{NIMBUS_USER}-${name}.esx.1  %{NIMBUS_USER}-${name}.esx.2  %{NIMBUS_USER}-${name}.nfs.0  %{NIMBUS_USER}-${name}.vc.0
-    Log To Console  Finished Creating Cluster ${name}
-
-    ${out}=  Get Static IP Address
-    Set Suite Variable  ${static}  ${out}
-    Append To List  ${list}  %{STATIC_WORKER_NAME}
+    Wait Until Keyword Succeeds  10x  10m  Create Simple VC Cluster With Static IP  ${name}
     
-    Log To Console  Set environment variables up for GOVC
-    Set Environment Variable  GOVC_URL  ${vc-ip}
-    Set Environment Variable  GOVC_USERNAME  Administrator@vsphere.local
-    Set Environment Variable  GOVC_PASSWORD  Admin\!23
-
-    Log To Console  Deploy VIC to the VC cluster
-    Set Environment Variable  TEST_URL_ARRAY  ${vc-ip}
-    Set Environment Variable  TEST_USERNAME  Administrator@vsphere.local
-    Set Environment Variable  TEST_PASSWORD  Admin\!23
-    Set Environment Variable  BRIDGE_NETWORK  bridge
-    Set Environment Variable  PUBLIC_NETWORK  vm-network
-    Remove Environment Variable  TEST_DATACENTER
-    Set Environment Variable  TEST_DATASTORE  nfs0-1
-    Set Environment Variable  TEST_RESOURCE  cls
-    Set Environment Variable  TEST_TIMEOUT  15m
-
 *** Test Cases ***
 Test
     Log To Console  \nStarting test...
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.md b/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.md
new file mode 100644
index 0000000000..bb870c5b38
--- /dev/null
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.md
@@ -0,0 +1,50 @@
+Test 5-28 - VIC Admin Isolated
+=======
+
+# Purpose:
+Verify that VIC Admin can display logs on an isolated network with no wan connection.
+
+# Environment:
+This test requires that a vSphere environment be running and available, which dSwitch test-ds available.
+
+# Test Steps:
+
+1. Create a DVS Port Group that does not have internet connectivity by setting a random vlan id
+2. Deploy VIC appliance to the vSphere server using the no-wan port group
+3. Verify login functions properly
+4. Verify vic admin internet connectivity status is showing a warning icon
+5. Pull the VCH-Init log and verify that it contains valid data
+6. Pull the Docker Personality log and verify that it contains valid data
+7. Create a container via the appliance
+8. Pull the container log bundle from the appliance and verify that it contains the new container's logs
+
+# Test Cases:
+
+## Display HTML
+1. Log in
+2. Page displays vic-machine name in title
+
+## WAN Status Should Fail
+1. Log in
+2. Page displays warning symbols for wan connection status
+
+## Fail To Pull Docker Image
+1. Log in
+2. Pull docker busybox
+3. Fail to pull busybox
+
+## Get Portlayer Log
+1. Log in
+2. Portlayer Log access is allowed and logs are downloaded
+
+## Get VCH-Init Log
+1. Log in
+2. VCH-Init Log access is allowed and logs are downloaded
+
+## Get Docker Personality Log
+1. Log in
+2. Docker Personality Log access is allowed and logs are downloaded
+
+## Get VICAdmin Log
+1. Log in
+2. VICAdmin Log access is allowed and logs are downloaded
\ No newline at end of file
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.robot
new file mode 100644
index 0000000000..bb7a9ccef7
--- /dev/null
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.robot
@@ -0,0 +1,99 @@
+# Copyright 2016-2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+*** Settings ***
+Documentation  Test 5-28 - VICAdmin Isolated
+Resource  ../../resources/Util.robot
+Suite Setup  Deploy Testbed With Static IP
+Suite Teardown  Teardown VCH With No WAN
+Default Tags
+
+*** Keywords ***
+Deploy Testbed With Static IP
+    Setup VC With No WAN
+    Deploy VCH With No WAN
+
+Setup VC With No WAN
+    ${name}=  Evaluate  'vic-5-28-' + str(random.randint(1000,9999))  modules=random
+    Wait Until Keyword Succeeds  10x  10m  Create Simple VC Cluster With Static IP  ${name}
+    Set Test Environment Variables
+    
+    Log To Console  Create a vch with a public network on a no-wan portgroup.
+
+    ${vlan}=  Evaluate  str(random.randint(1, 195))  modules=random
+
+    ${dvs}=  Run  govc find -type DistributedVirtualSwitch | head -n1
+    ${rc}  ${output}=  Run And Return Rc And Output  govc dvs.portgroup.add -vlan=${vlan} -dvs ${dvs} dpg-no-wan
+    Should Be Equal As Integers  ${rc}  0
+    
+Deploy VCH With No WAN
+    ${out}=  Deploy VCH With No WAN Secret
+    Log  ${out}
+    Get Docker Params  ${out}  ${false}
+
+Deploy VCH With No WAN Secret
+    [Tags]  secret
+    ${output}=  Run  bin/vic-machine-linux create --debug 1 --name=%{VCH-NAME} --target=%{TEST_URL}%{TEST_DATACENTER} --thumbprint=%{TEST_THUMBPRINT} --user=%{TEST_USERNAME} --image-store=%{TEST_DATASTORE} --password=%{TEST_PASSWORD} --force=true --compute-resource=%{TEST_RESOURCE} --no-tlsverify --bridge-network=%{BRIDGE_NETWORK} --management-network=%{PUBLIC_NETWORK} --client-network=%{PUBLIC_NETWORK} --client-network-ip &{static}[ip]/&{static}[netmask] --client-network-gateway 10.0.0.0/8:&{static}[gateway] --public-network dpg-no-wan --public-network-ip 192.168.100.2/24 --public-network-gateway 192.168.100.1 --dns-server 10.170.16.48 --insecure-registry wdc-harbor-ci.eng.vmware.com
+    [Return]  ${output}
+
+Teardown VCH With No WAN
+    Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
+
+Login And Save Cookies
+    [Tags]  secret
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -sk %{VIC-ADMIN}/authentication -XPOST -F username=%{TEST_USERNAME} -F password=%{TEST_PASSWORD} -D /tmp/cookies-%{VCH-NAME}
+    Should Be Equal As Integers  ${rc}  0
+
+*** Test Cases ***
+Display HTML
+    Login And Save Cookies
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -sk %{VIC-ADMIN} -b /tmp/cookies-%{VCH-NAME}
+    Log  ${output}
+    Should contain  ${output}  <title>VIC: %{VCH-NAME}</title>
+
+WAN Status Should Fail
+    Login And Save Cookies
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -sk %{VIC-ADMIN} -b /tmp/cookies-%{VCH-NAME}
+    Log  ${output}
+    Should contain  ${output}  <div class="sixty">Registry and Internet Connectivity<span class="error-message">
+
+Fail To Pull Docker Image
+    Login And Save Cookies
+    ${rc}  ${output}=  Run And Return Rc and Output  docker %{VCH-PARAMS} pull ${busybox}
+    Log  ${output}
+    Should Be Equal As Integers  ${rc}  1
+    Should contain  ${output}  no route to host
+
+Get Portlayer Log
+    Login And Save Cookies
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -sk %{VIC-ADMIN}/logs/port-layer.log -b /tmp/cookies-%{VCH-NAME}
+    Log  ${output}
+    Should contain  ${output}  Launching portlayer server
+
+Get VCH-Init Log
+    Login And Save Cookies
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -sk %{VIC-ADMIN}/logs/init.log -b /tmp/cookies-%{VCH-NAME}
+    Log  ${output}
+    Should contain  ${output}  reaping child processes
+
+Get Docker Personality Log
+    Login And Save Cookies
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -sk %{VIC-ADMIN}/logs/docker-personality.log -b /tmp/cookies-%{VCH-NAME}
+    Log  ${output}
+    Should contain  ${output}  docker personality
+
+Get VICAdmin Log
+    Login And Save Cookies
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -sk %{VIC-ADMIN}/logs/vicadmin.log -b /tmp/cookies-%{VCH-NAME}
+    Log  ${output}
+    Should contain  ${output}  Launching vicadmin pprof server
\ No newline at end of file
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/TestCases.md b/tests/manual-test-cases/Group5-Functional-Tests/TestCases.md
index 2eb7e414c3..66cfa3a767 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/TestCases.md
+++ b/tests/manual-test-cases/Group5-Functional-Tests/TestCases.md
@@ -52,4 +52,4 @@ Group 5 - Functional Tests
 -
 [Test 5-27 - Selenium Grid](5-27-Selenium-Grid.md)
 -
-
+[Test 5-28 - VICAdmin Isolated](5-28-VICAdmin-Isolated.md)
diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index ac8c2992ed..fcbc733d4f 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -451,6 +451,45 @@ Power Off Host
     ${out}=  Execute Command  poweroff -d 0 -f
     Close connection
 
+Create Simple VC Cluster With Static IP
+    [Arguments]  ${name}=vic-simple-vc-static-ip
+    [Timeout]    110 minutes
+    Set Suite Variable  ${NIMBUS_LOCATION}  NIMBUS_LOCATION=wdc
+    Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
+    Log To Console  Create a new simple vc cluser with static ip support...
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-simple-cluster.rb --runName ${name}
+    Log  ${out}
+
+    Open Connection  %{NIMBUS_GW}
+    Wait Until Keyword Succeeds  10 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
+    ${vc-ip}=  Get IP  ${name}.vc.0
+    ${pod}=  Fetch POD  ${name}.vc.0
+    Set Suite Variable  ${NIMBUS_POD}  ${pod}
+    Close Connection
+
+    Set Suite Variable  @{list}  %{NIMBUS_USER}-${name}.esx.0  %{NIMBUS_USER}-${name}.esx.1  %{NIMBUS_USER}-${name}.esx.2  %{NIMBUS_USER}-${name}.nfs.0  %{NIMBUS_USER}-${name}.vc.0
+    Log To Console  Finished Creating Cluster ${name}
+
+    ${out}=  Get Static IP Address
+    Set Suite Variable  ${static}  ${out}
+    Append To List  ${list}  %{STATIC_WORKER_NAME}
+
+    Log To Console  Set environment variables up for GOVC
+    Set Environment Variable  GOVC_URL  ${vc-ip}
+    Set Environment Variable  GOVC_USERNAME  Administrator@vsphere.local
+    Set Environment Variable  GOVC_PASSWORD  Admin\!23
+
+    Log To Console  Deploy VIC to the VC cluster
+    Set Environment Variable  TEST_URL_ARRAY  ${vc-ip}
+    Set Environment Variable  TEST_USERNAME  Administrator@vsphere.local
+    Set Environment Variable  TEST_PASSWORD  Admin\!23
+    Set Environment Variable  BRIDGE_NETWORK  bridge
+    Set Environment Variable  PUBLIC_NETWORK  vm-network
+    Remove Environment Variable  TEST_DATACENTER
+    Set Environment Variable  TEST_DATASTORE  nfs0-1
+    Set Environment Variable  TEST_RESOURCE  cls
+    Set Environment Variable  TEST_TIMEOUT  15m
+
 Create Static IP Worker
     Open Connection  %{NIMBUS_GW}
     Wait Until Keyword Succeeds  10 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}

From 22662a588441d1665b6b210d011aad672e016aca Mon Sep 17 00:00:00 2001
From: Zach Tucker <jzt@users.noreply.github.com>
Date: Tue, 24 Apr 2018 20:43:16 -0500
Subject: [PATCH 05/31] Fix setup/teardown in opsuser tests (#7836) (#8151)

(cherry picked from commit c6ea3e26caeefcf0e2df7c842824492e64424885)
---
 .../Group5-Functional-Tests/5-25-OPS-User-Grant.robot      | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
index e3965a3765..4a62f819a1 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
@@ -15,10 +15,9 @@
 *** Settings ***
 Documentation  Test 5-25 - OPS-User-Grant
 Resource  ../../resources/Util.robot
-Suite Setup  Ops User Create
-#Suite Setup  Wait Until Keyword Succeeds  10x  10m  Ops User Create
-#Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
-#Test Teardown  Run Keyword If Test Failed  Gather vSphere Logs
+Suite Setup  Wait Until Keyword Succeeds  10x  10m  Ops User Create
+Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
+Test Teardown  Run Keyword If Test Failed  Gather vSphere Logs
 
 *** Keywords ***
 Ops User Create

From 26a741aa11c2876e1f22783ce2c265435fbe99bd Mon Sep 17 00:00:00 2001
From: Zach Shepherd <zjs@users.noreply.github.com>
Date: Tue, 1 May 2018 09:17:11 -0700
Subject: [PATCH 06/31] nightly: Disable 5-25-OPS-User-Grant upgrade test
 (#7863) (#8151)

Disable the "granted ops-user perms work after upgrade" test in the
5-25-OPS-User-Grant nightly suite.

(cherry picked from commit 7b6cde7405456ae77c4fd01433f8303f45356d24)
---
 .../5-25-OPS-User-Grant.robot                 | 30 +++++++++++--------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
index 4a62f819a1..e70b1ef5b0 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
@@ -142,20 +142,24 @@ vic-machine create grants ops-user perms
     Cleanup VIC Appliance On Test Server
 
 granted ops-user perms work after upgrade
-    Install VIC with version to Test Server  v1.3.0  additional-args=--ops-user ${ops_user_name} --ops-password ${ops_user_password} --ops-grant-perms
-
-    Check Original Version
-    Upgrade
-    Check Upgraded Version
-
+    ${status}=  Get State Of Github Issue  7796
+    Run Keyword If  '${status}' == 'closed'  Fail  Test 5-25-OPS-User-Grant.robot needs to be updated now that Issue #7796 has been resolved
+    Log  Issue \#7796 is blocking implementation  WARN
+
+    #Install VIC with version to Test Server  v1.3.0  additional-args=--ops-user ${ops_user_name} --ops-password ${ops_user_password} --ops-grant-perms
+    #
+    #Check Original Version
+    #Upgrade
+    #Check Upgraded Version
+    #
     # Run a govc test to check that access is denied on some resources
-    Attempt To Create Resource Pool
-
-    Run Regression Tests
-
-    Run privilege-dependent docker operations
-
-    Cleanup VIC Appliance On Test Server
+    #Attempt To Create Resource Pool
+    #
+    #Run Regression Tests
+    #
+    #Run privilege-dependent docker operations
+    #
+    #Cleanup VIC Appliance On Test Server
 
 Test with VM-Host Affinity
     Log To Console  \nStarting test...

From 41140c93e801f7a71dc6913e8322cda510c2be6c Mon Sep 17 00:00:00 2001
From: rashok-vmware <rashok@vmware.com>
Date: Tue, 1 May 2018 12:06:52 -0700
Subject: [PATCH 07/31] Nimbus testbed for nfs volume nightly test (#7853)
 (#8151)

Updated the NFS Volume nightlies to build a full NFS test bed instead of relying on the unsupported pod placement calls.

Fixes #7642

(cherry picked from commit ff6baa913ce76994460f4e592a4e866a6f5c2177)
---
 .../5-22-NFS-Volume.robot                     | 18 ++------
 tests/resources/Nimbus-Util.robot             | 30 +++++++------
 tests/resources/nimbus-testbeds/vic-nfs.rb    | 43 +++++++++++++++++++
 3 files changed, 63 insertions(+), 28 deletions(-)
 create mode 100644 tests/resources/nimbus-testbeds/vic-nfs.rb

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
index 76f609b074..43b7ca725e 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
@@ -38,23 +38,14 @@ Setup ESX And NFS Suite
     Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
     Log To Console  \nStarting test...
 
-    ${esx1}  ${esx1_ip}=  Deploy Nimbus ESXi Server  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
-    Open Connection  %{NIMBUS_GW}
-    Wait Until Keyword Succeeds  2 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
-    ${POD}=  Fetch POD  ${esx1}
-    Log To Console  ${POD}
-    Close Connection
-
-    ${nfs}  ${nfs_ip}=  Deploy Nimbus NFS Datastore  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  additional-args=--nimbus ${POD}
+    ${nfs}  ${nfs_ro}  ${esx1}  ${nfs_ip}  ${nfs_ro_ip}  ${esx1_ip}=  Deploy Simple NFS Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  additional-args=--testbedSpecRubyFile /dbc/w3-dbc302/rashok/vic-nfs.rb --esxBuild ${ESX_VERSION} --esxPxeDir ${ESX_VERSION} --plugin testng
 
-    ${nfs_readonly}  ${nfs_readonly_ip}=  Deploy Nimbus NFS Datastore  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  additional-args=--disk 5000000 --disk 5000000 --mountOpt ro --nfsOpt ro --mountPoint=storage1 --mountPoint=storage2 --nimbus ${POD}
-
-    Set Suite Variable  @{list}  ${esx1}  ${nfs}  ${nfs_readonly}
+    Set Suite Variable  @{list}  ${esx1}  ${nfs}  ${nfs_ro}
     Set Suite Variable  ${ESX1}  ${esx1}
     Set Suite Variable  ${ESX1_IP}  ${esx1_ip}
     Set Suite Variable  ${NFS_IP}  ${nfs_ip}
     Set Suite Variable  ${NFS}  ${nfs}
-    Set Suite Variable  ${NFS_READONLY_IP}  ${nfs_readonly_ip}
+    Set Suite Variable  ${NFS_READONLY_IP}  ${nfs_ro_ip}
 
     # Add the NFS servers to SSH known host list
     ${out}=  Run Keyword And Ignore Error  Run  sshpass -p %{DEPLOYED_PASSWORD} ssh -o StrictHostKeyChecking\=no root@${NFS_IP} exit
@@ -84,7 +75,7 @@ Setup ENV Variables for VIC Appliance Install
     Set Environment Variable  TEST_URL  ${ESX1_IP}
     Set Environment Variable  TEST_USERNAME  root
     Set Environment Variable  TEST_PASSWORD  ${NIMBUS_ESX_PASSWORD}
-    Set Environment Variable  TEST_DATASTORE  datastore1
+    Set Environment Variable  TEST_DATASTORE  local-0
     Set Environment Variable  TEST_TIMEOUT  30m
     Set Environment Variable  HOST_TYPE  ESXi
     Remove Environment Variable  TEST_DATACENTER
@@ -329,7 +320,6 @@ Docker Inspect Mount Data after Reboot
 
     Verify Volume Inspect Info  After VM Reboot  ${mntDataTestContainer}  ${checkList}
 
-
 Kill NFS Server
     Sleep  5 minutes
     ${rc}  ${runningContainer}=  Run And Return Rc And Output  docker %{VCH-PARAMS} run -d -v ${nfsNamedVolume}:/mydata ${busybox} sh -c "while true; do echo 'Still here...\n' >> /mydata/test_nfs_kill.txt; sleep 2; done"
diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index fcbc733d4f..248580381b 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -402,28 +402,30 @@ Get Vsphere Version
     \   ${status}=  Run Keyword And Return Status  Should Contain  ${line}  Version:
     \   Run Keyword And Return If  ${status}  Fetch From Right  ${line}  ${SPACE}
 
-Deploy Nimbus NFS Datastore
+Deploy Simple NFS Testbed
     [Arguments]  ${user}  ${password}  ${additional-args}=
     ${name}=  Evaluate  'NFS-' + str(random.randint(1000,9999)) + str(time.clock())  modules=random,time
-    Log To Console  \nDeploying Nimbus NFS server: ${name}
+    Log To Console  \nDeploying Nimbus NFS testbed: ${name}
     Open Connection  %{NIMBUS_GW}
     Wait Until Keyword Succeeds  2 min  30 sec  Login  ${user}  ${password}
 
-    ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-nfsdeploy ${name} ${additional-args}
+    ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-testbeddeploy --testbedName nfs --runName ${name} ${additional-args}
     Log  ${out}
     # Make sure the deploy actually worked
-    Should Contain  ${out}  To manage this VM use
-    # Now grab the IP address and return the name and ip for later use
-    @{out}=  Split To Lines  ${out}
-    :FOR  ${item}  IN  @{out}
-    \   ${status}  ${message}=  Run Keyword And Ignore Error  Should Contain  ${item}  IP is
-    \   Run Keyword If  '${status}' == 'PASS'  Set Suite Variable  ${line}  ${item}
-    @{gotIP}=  Split String  ${line}  ${SPACE}
-    ${ip}=  Remove String  @{gotIP}[5]  ,
+    Should Contain  ${out}  ${name}.nfs.0' is up. IP:
 
-    Log To Console  Successfully deployed new NFS server - ${user}-${name}
-    Close connection
-    [Return]  ${user}-${name}  ${ip}
+    Open Connection  %{NIMBUS_GW}
+    Wait Until Keyword Succeeds  10 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
+    ${nfs-ip}=  Get IP  ${name}.nfs.0
+    ${nfs-ro-ip}=  Get IP  ${name}.nfs.1
+    ${esx-ip}=  Get IP  ${name}.esx.0
+    Close Connection
+
+    Log To Console  \nNFS IP: ${nfs-ip}
+    Log To Console  \nNFS READ-Only IP: ${nfs-ro-ip}
+    Log To Console  \nESX IP: ${esx-ip}
+
+    [Return]  ${user}-${name}.nfs.0  ${user}-${name}.nfs.1  ${user}-${name}.esx.0  ${nfs-ip}  ${nfs-ro-ip}  ${esx-ip}
 
 Change ESXi Server Password
     [Arguments]  ${password}
diff --git a/tests/resources/nimbus-testbeds/vic-nfs.rb b/tests/resources/nimbus-testbeds/vic-nfs.rb
new file mode 100644
index 0000000000..4954bc2999
--- /dev/null
+++ b/tests/resources/nimbus-testbeds/vic-nfs.rb
@@ -0,0 +1,43 @@
+require '/mts/git/nimbus/lib/testframeworks/testng/testng.rb'
+oneGB = 1 * 1000 * 1000
+$testbed = Proc.new do |sharedStorageStyle, esxStyle|
+  esxStyle ||= 'fullInstall'
+  esxStyle = esxStyle.to_s
+  sharedStorageStyle = sharedStorageStyle.to_s
+
+  testbed = {
+    'version' => 3,
+    'name' => "nfs-1esx-#{sharedStorageStyle}-#{esxStyle}",
+    'esx' => [
+      {
+        'name' => "esx.0",
+        'style' => esxStyle,
+        'numMem' => 16 * 1024,
+        'numCPUs' => 6,
+        "disks" => [ 30 * oneGB, 30 * oneGB, 30 * oneGB],
+        'disableNfsMounts' => true,
+        'nics' => 2,
+        'staf' => false,
+        'desiredPassword' => 'e2eFunctionalTest',
+    }],
+    'nfs' => [{
+    'name' => 'dev-nfs',
+    'type' => 'NFS',
+    'nfsOpt' => 'rw',
+    'numMem' => 16 * 1024,
+    },
+    'nfs' => {
+    'name' => 'dev-nfs',
+    'type' => 'NFS',
+    'nfsOpt' => 'ro',
+    'numMem' => 16 * 1024,
+    }],
+  }
+end
+
+[:pxeBoot, :fullInstall].each do |esxStyle|
+    [:iscsi, :fc].each do |sharedStorageStyle|
+      testbedSpec = $testbed.call(sharedStorageStyle, esxStyle)
+      Nimbus::TestbedRegistry.registerTestbed testbedSpec
+    end
+  end

From 7e7a3512c61f5cf32feb4d34a037acdfe0748de0 Mon Sep 17 00:00:00 2001
From: Zach Tucker <jzt@users.noreply.github.com>
Date: Tue, 1 May 2018 18:13:21 +0000
Subject: [PATCH 08/31] Use 1.3.1 in ops user upgrade test (#8151)

(cherry picked from commit 47851e3cf6f1b199f7f59fdbfa12d4d44c93ba9d)
---
 .../5-25-OPS-User-Grant.md                    |  2 +-
 .../5-25-OPS-User-Grant.robot                 | 37 ++++++++-----------
 2 files changed, 17 insertions(+), 22 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.md b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.md
index d31b8a6e6c..577fe59a89 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.md
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.md
@@ -17,7 +17,7 @@ This test requires access to VMware Nimbus cluster for dynamic ESXi and vCenter
 7. Create a container
 8. Use govc to attempt to out-of-band destroy the container from Step 7
 9. Clean up the VCH
-10. Install version v1.3.0 of the VIC appliance into the cluster with the --ops-grant-perms option
+10. Install version v1.3.1 of the VIC appliance into the cluster with the --ops-grant-perms option
 11. Perform a VCH upgrade to the current version
 12. With the ops-user, use govc to attempt to create a resource pool
 13. Run a variety of docker operations on the VCH
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
index e70b1ef5b0..4bd7806b96 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
@@ -17,7 +17,6 @@ Documentation  Test 5-25 - OPS-User-Grant
 Resource  ../../resources/Util.robot
 Suite Setup  Wait Until Keyword Succeeds  10x  10m  Ops User Create
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
-Test Teardown  Run Keyword If Test Failed  Gather vSphere Logs
 
 *** Keywords ***
 Ops User Create
@@ -139,27 +138,23 @@ vic-machine create grants ops-user perms
 
     Run privilege-dependent docker operations
 
-    Cleanup VIC Appliance On Test Server
+    [Teardown]  Cleanup VIC Appliance On Test Server
 
 granted ops-user perms work after upgrade
-    ${status}=  Get State Of Github Issue  7796
-    Run Keyword If  '${status}' == 'closed'  Fail  Test 5-25-OPS-User-Grant.robot needs to be updated now that Issue #7796 has been resolved
-    Log  Issue \#7796 is blocking implementation  WARN
-
-    #Install VIC with version to Test Server  v1.3.0  additional-args=--ops-user ${ops_user_name} --ops-password ${ops_user_password} --ops-grant-perms
-    #
-    #Check Original Version
-    #Upgrade
-    #Check Upgraded Version
-    #
+    Install VIC with version to Test Server  v1.3.1  additional-args=--ops-user ${ops_user_name} --ops-password ${ops_user_password} --ops-grant-perms
+
+    Check Original Version
+    Upgrade
+    Check Upgraded Version
+
     # Run a govc test to check that access is denied on some resources
-    #Attempt To Create Resource Pool
-    #
-    #Run Regression Tests
-    #
-    #Run privilege-dependent docker operations
-    #
-    #Cleanup VIC Appliance On Test Server
+    Attempt To Create Resource Pool
+
+    Run Regression Tests
+
+    Run privilege-dependent docker operations
+
+    [Teardown]  Cleanup VIC Appliance On Test Server
 
 Test with VM-Host Affinity
     Log To Console  \nStarting test...
@@ -172,7 +167,7 @@ Test with VM-Host Affinity
 
     Run privilege-dependent docker operations
 
-    Cleanup VIC Appliance On Test Server
+    [Teardown]  Cleanup VIC Appliance On Test Server
 
 vic-machine configure grants ops-user perms
     Install VIC Appliance To Test Server
@@ -184,4 +179,4 @@ vic-machine configure grants ops-user perms
 
     Run privilege-dependent docker operations
 
-    Cleanup VIC Appliance On Test Server
+    [Teardown]  Cleanup VIC Appliance On Test Server

From 3057da1ac9941f88ae7b0e107f6007e995240074 Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Wed, 2 May 2018 14:07:59 -0500
Subject: [PATCH 09/31] Fix several tests that did not appropriately use the
 vic-admin env var (#7879) (#8151)

(cherry picked from commit b5bad944c13aa15f4c2dd1790ad7a1479969229a)
---
 .../Group23-VIC-Machine-Service/23-02-VCH-List.robot  | 11 -----------
 .../Group24-Multi-VCH/24-01-Multi-VCH-PS.robot        |  2 ++
 .../Group24-Multi-VCH/24-02-Multi-VCH-Delete.robot    |  3 +++
 tests/test-cases/Group6-VIC-Machine/6-13-TLS.robot    |  3 +++
 4 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/tests/test-cases/Group23-VIC-Machine-Service/23-02-VCH-List.robot b/tests/test-cases/Group23-VIC-Machine-Service/23-02-VCH-List.robot
index 4a4677f879..5003721343 100644
--- a/tests/test-cases/Group23-VIC-Machine-Service/23-02-VCH-List.robot
+++ b/tests/test-cases/Group23-VIC-Machine-Service/23-02-VCH-List.robot
@@ -121,14 +121,3 @@ Get VCH List Within Invalid Datacenter and Compute Resource
 
     Verify Return Code
     Verify Status Not Found
-
-
-Get Empty VCH List When No VCH deployed
-    Cleanup VIC Appliance On Test Server
-
-    Get VCH List
-
-    Verify Return Code
-    Verify Status Ok
-
-    Verify VCH List Empty
diff --git a/tests/test-cases/Group24-Multi-VCH/24-01-Multi-VCH-PS.robot b/tests/test-cases/Group24-Multi-VCH/24-01-Multi-VCH-PS.robot
index 298e7f6e5d..0c3261c46b 100644
--- a/tests/test-cases/Group24-Multi-VCH/24-01-Multi-VCH-PS.robot
+++ b/tests/test-cases/Group24-Multi-VCH/24-01-Multi-VCH-PS.robot
@@ -23,6 +23,7 @@ Clean Up VCHs
     Set Environment Variable  VCH-NAME  ${old-vm}
     Set Environment Variable  BRIDGE_NETWORK  ${old-vch-bridge}
     Set Environment Variable  VCH-PARAMS  ${old-vch-params}
+    Set Environment Variable  VIC-ADMIN  ${old-vic-admin}
     Run Keyword And Continue On Failure  Cleanup VIC Appliance On Test Server
 
 *** Test Cases ***
@@ -34,6 +35,7 @@ Create Multi VCH - Docker Ps Only Contains The Correct Containers
     Set Suite Variable  ${old-vm}  %{VCH-NAME}
     Set Suite Variable  ${old-vch-params}  %{VCH-PARAMS}
     Set Suite Variable  ${old-vch-bridge}  %{BRIDGE_NETWORK}
+    Set Suite Variable  ${old-vic-admin}  %{VIC-ADMIN}
 
     # make sure we create two different bridge networks
     Remove Environment Variable  BRIDGE_NETWORK
diff --git a/tests/test-cases/Group24-Multi-VCH/24-02-Multi-VCH-Delete.robot b/tests/test-cases/Group24-Multi-VCH/24-02-Multi-VCH-Delete.robot
index 2a5bb2fcf6..33be3c935e 100644
--- a/tests/test-cases/Group24-Multi-VCH/24-02-Multi-VCH-Delete.robot
+++ b/tests/test-cases/Group24-Multi-VCH/24-02-Multi-VCH-Delete.robot
@@ -23,6 +23,7 @@ Cleanup VCHs
     Run Keyword If  '${old-vch}' != '${EMPTY}'  Set Environment Variable  VCH-NAME  ${old-vch}
     Run Keyword If  '${old-vch-bridge}' != '${EMPTY}'  Set Environment Variable  BRIDGE_NETWORK  ${old-vch-bridge}
     Run Keyword If  '${old-vch-params}' != '${EMPTY}'  Set Environment Variable  VCH-PARAMS  ${old-vch-params}
+    Run Keyword If  '${old-vic-admin}' != '${EMPTY}'  Set Environment Variable  VIC-ADMIN  ${old-vic-admin}
     Run Keyword And Continue On Failure  Cleanup VIC Appliance On Test Server
 
 *** Test Cases ***
@@ -40,6 +41,7 @@ VCH delete only removes its own containers
     Set Suite Variable  ${old-vch}  %{VCH-NAME}
     Set Suite Variable  ${old-vch-params}  %{VCH-PARAMS}
     Set Suite Variable  ${old-vch-bridge}  %{BRIDGE_NETWORK}
+    Set Suite Variable  ${old-vic-admin}  %{VIC-ADMIN}
 
     # Unset BRIDGE_NETWORK so the new VCH uses a unique bridge network
     Remove Environment Variable  BRIDGE_NETWORK
@@ -64,4 +66,5 @@ VCH delete only removes its own containers
     Set Environment Variable  VCH-NAME  ${old-vch}
     Set Environment Variable  BRIDGE_NETWORK  ${old-vch-bridge}
     Set Environment Variable  VCH-PARAMS  ${old-vch-params}
+    Set Environment Variable  VIC-ADMIN  ${old-vic-admin}
     Cleanup VIC Appliance On Test Server
\ No newline at end of file
diff --git a/tests/test-cases/Group6-VIC-Machine/6-13-TLS.robot b/tests/test-cases/Group6-VIC-Machine/6-13-TLS.robot
index 870ca139dd..28521036cd 100644
--- a/tests/test-cases/Group6-VIC-Machine/6-13-TLS.robot
+++ b/tests/test-cases/Group6-VIC-Machine/6-13-TLS.robot
@@ -123,6 +123,7 @@ Create VCH - Server certificate with multiple blocks
     Should Contain  ${output}  Failed to load x509 leaf
     Should Contain  ${output}  Loaded server certificate
     Should Contain  ${output}  Installer completed successfully
+    Get Docker Params  ${output}  ${true}
 
     Cleanup VIC Appliance On Test Server
 
@@ -137,6 +138,7 @@ Create VCH - Invalid keys
     Run Keyword And Ignore Error  Cleanup Datastore On Test Server
 
     ${output}=  Run  bin/vic-machine-linux create --name=%{VCH-NAME} --target="%{TEST_USERNAME}:%{TEST_PASSWORD}@%{TEST_URL}" --thumbprint=%{TEST_THUMBPRINT} --image-store=%{TEST_DATASTORE} --bridge-network=%{BRIDGE_NETWORK} --public-network=%{PUBLIC_NETWORK} ${vicmachinetls}
+    Get Docker Params  ${output}  ${true}
 
     # Invalid server key
     ${output}=  Run  bin/vic-machine-linux create --name=%{VCH-NAME} --target="%{TEST_USERNAME}:%{TEST_PASSWORD}@%{TEST_URL}" --thumbprint=%{TEST_THUMBPRINT} --image-store=%{TEST_DATASTORE} --bridge-network=%{BRIDGE_NETWORK} --public-network=%{PUBLIC_NETWORK} ${vicmachinetls} --tls-ca="./%{VCH-NAME}/ca.pem" --tls-server-cert="./%{VCH-NAME}/server-cert.pem" --tls-server-key="./%{VCH-NAME}/ca.pem"
@@ -177,6 +179,7 @@ Create VCH - Reuse keys
     Should Contain  ${output}  Loaded server certificate
     Should Contain  ${output}  Loaded CA with default name from certificate path
     Should Contain  ${output}  Loaded client certificate with default name from certificate path
+    Get Docker Params  ${output}  ${true}
 
     Cleanup VIC Appliance On Test Server
 

From b8c0173591ed9eb805554c506e6059b350ca5e7b Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Thu, 3 May 2018 11:45:02 -0500
Subject: [PATCH 10/31] Update the nightlies builds (#7883) (#8151)

(cherry picked from commit 70e4c2b2e3c5122be23f5233aad08b6ae4ba2e09)
---
 tests/nightly/jenkins-nightly-run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh
index d54908ca08..8ac0c06b18 100755
--- a/tests/nightly/jenkins-nightly-run.sh
+++ b/tests/nightly/jenkins-nightly-run.sh
@@ -16,8 +16,8 @@
 ESX_60_VERSION="ob-5251623"
 VC_60_VERSION="ob-5112509"
 
-ESX_67_VERSION="ob-8126342"
-VC_67_VERSION="ob-8130682"
+ESX_67_VERSION="ob-8169922"
+VC_67_VERSION="ob-8217866"
 
 if [[ $1 != "6.0" && $1 != "6.5" && $1 != "6.7" ]]; then
     echo "Please specify a target cluster. One of: 6.0, 6.5, 6.7"

From f30e51700fb68edf71b7c2cc689acf5499b7eb4e Mon Sep 17 00:00:00 2001
From: Anchal Agrawal <anchala@vmware.com>
Date: Thu, 3 May 2018 11:55:07 -0500
Subject: [PATCH 11/31] Configure ROBO nightlies with Group19-DRS-Disabled
 (#7876) (#8151)

This commit adds test scripts for running a nightly Jenkins job to
run tests in a non-DRS environment. This change runs the
Group19-DRS-Disabled suite against a pre-deployed, static vCenter
ELM testbed. The corresponding secrets file will be committed to
the internal repo.

Future changes will include adding more tests and using a testbed
deployed with a service account.

Fixes #7662

(cherry picked from commit fe040e063b6fca04bd68b1b99f9377d444127b65)
---
 .../drs-disabled/jenkins-drs-disabled-run.sh  | 81 +++++++++++++++++++
 tests/drs-disabled/upload-logs.sh             | 56 +++++++++++++
 .../19-3-DRS-Disabled-Placement.robot         |  9 ++-
 3 files changed, 142 insertions(+), 4 deletions(-)
 create mode 100755 tests/drs-disabled/jenkins-drs-disabled-run.sh
 create mode 100755 tests/drs-disabled/upload-logs.sh

diff --git a/tests/drs-disabled/jenkins-drs-disabled-run.sh b/tests/drs-disabled/jenkins-drs-disabled-run.sh
new file mode 100755
index 0000000000..442fbe3975
--- /dev/null
+++ b/tests/drs-disabled/jenkins-drs-disabled-run.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+# Copyright 2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+input=$(gsutil ls -l gs://vic-engine-builds/vic_* | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | cut -d '/' -f 4)
+
+n=0
+   until [ $n -ge 5 ]
+   do
+      echo "Retry.. $n"
+      echo "Downloading gcp file $input"
+      wget https://storage.googleapis.com/vic-engine-builds/$input
+      if [ -f "$input" ]
+      then
+      echo "File found.."
+      break
+      else
+      echo "File NOT found"
+      fi
+      n=$[$n+1]
+      sleep 15
+   done
+
+n=0
+   until [ $n -ge 5 ]
+   do
+      mkdir bin
+      echo "Extracting .tar.gz"
+      tar xvzf $input -C bin/ --strip 1
+      if [ -f "bin/vic-machine-linux" ]
+      then
+      echo "tar extraction complete.."
+      canContinue="Yes"
+      break
+      else
+      echo "tar extraction failed"
+      canContinue="No"
+      rm -rf bin
+      fi
+      n=$[$n+1]
+      sleep 15
+   done
+
+if [[ $canContinue = "No" ]]; then
+    echo "Tarball extraction failed..quitting the run"
+    break
+else
+    echo "Tarball extraction passed, Running nightlies test.."
+fi
+
+# Run the Robot tests in a container
+envfile="vic-internal/drs-disabled-secrets.list"
+image="gcr.io/eminent-nation-87317/vic-integration-test:1.48"
+cmd="pabot --processes 1 --removekeywords TAG:secret -d drs-disabled tests/manual-test-cases/Group19-DRS-Disabled"
+docker run --rm -v $PWD/vic:/go --env-file $envfile $image $cmd
+
+# Remove the VIC binary tar file
+rm $input
+
+cat vic/drs-disabled/pabot_results/*/stdout.txt | grep -E '::|\.\.\.' | grep -E 'PASS|FAIL' > console.log
+
+# Format the email output for Jenkins
+sed -i -e 's/^/<br>/g' console.log
+sed -i -e 's|PASS|<font color="green">PASS</font>|g' console.log
+sed -i -e 's|FAIL|<font color="red">FAIL</font>|g' console.log
+
+# Run the log upload script in a container
+cmd="tests/drs-disabled/upload-logs.sh"
+docker run --rm -v $PWD/vic:/go --env-file $envfile $image $cmd
+
diff --git a/tests/drs-disabled/upload-logs.sh b/tests/drs-disabled/upload-logs.sh
new file mode 100755
index 0000000000..83f98a167e
--- /dev/null
+++ b/tests/drs-disabled/upload-logs.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Copyright 2016-2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo "Upload drs-disabled test logs"
+
+set -x
+gsutil version -l
+set +x
+
+outfile="vic_drs_disabled_logs_"$BUILD_TIMESTAMP".zip"
+echo $outfile
+
+/usr/bin/zip -9 -r $outfile drs-disabled *.zip *.log *.debug *.tgz
+
+# GC credentials
+keyfile=~/vic-drs-disabled-logs.key
+botofile=~/.boto
+if [ ! -f $keyfile ]; then
+    echo -en $GS_PRIVATE_KEY > $keyfile
+    chmod 400 $keyfile
+fi
+if [ ! -f $botofile ]; then
+    echo "[Credentials]" >> $botofile
+    echo "gs_service_key_file = $keyfile" >> $botofile
+    echo "gs_service_client_id = $GS_CLIENT_EMAIL" >> $botofile
+    echo "[GSUtil]" >> $botofile
+    echo "content_language = en" >> $botofile
+    echo "default_project_id = $GS_PROJECT_ID" >> $botofile
+fi
+
+if [ -f "$outfile" ]; then
+  gsutil cp $outfile gs://vic-drs-disabled-logs
+  echo "----------------------------------------------"
+  echo "Download test logs here:"
+  echo "https://console.cloud.google.com/m/cloudstorage/b/vic-drs-disabled-logs/o/$outfile?authuser=1"
+  echo "----------------------------------------------"
+else
+  echo "No log output file to upload"
+fi
+
+if [ -f "$keyfile" ]; then
+  rm -f $keyfile
+fi
diff --git a/tests/manual-test-cases/Group19-DRS-Disabled/19-3-DRS-Disabled-Placement.robot b/tests/manual-test-cases/Group19-DRS-Disabled/19-3-DRS-Disabled-Placement.robot
index b077b0d839..e7d3a77038 100644
--- a/tests/manual-test-cases/Group19-DRS-Disabled/19-3-DRS-Disabled-Placement.robot
+++ b/tests/manual-test-cases/Group19-DRS-Disabled/19-3-DRS-Disabled-Placement.robot
@@ -49,13 +49,14 @@ Teardown VCH And Cleanup Nimbus
     Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Test Cases ***
-
 # TODO(jzt): we need to test against a single ESX host
 
 Simple Placement
-    Set Environment Variable  GOVC_URL  ${vc1-ip}
-    Set Environment Variable  TEST_URL_ARRAY  ${vc1-ip}
-    Set Environment Variable  TEST_RESOURCE  cls3
+    # TODO(anchal): these are currently set to the static testbed in the secrets file.
+    # Set Environment Variable  GOVC_URL  ${vc1-ip}
+    # Set Environment Variable  TEST_URL_ARRAY  ${vc1-ip}
+    # Set Environment Variable  TEST_RESOURCE  cls3
+
     Set Environment Variable  TEST_TIMEOUT  30m
 
     Log To Console  Deploy VIC to the VC cluster

From b4632246f5326f0e86ce6535636320dcbda87250 Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Thu, 3 May 2018 11:55:54 -0500
Subject: [PATCH 12/31] Add timeouts in case of hung operations (#7884) (#8151)

(cherry picked from commit 72391e5fb37a24fb953eeaf0eb358175247cfa8a)
---
 .../Group5-Functional-Tests/5-25-OPS-User-Grant.robot            | 1 +
 tests/resources/Docker-Util.robot                                | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
index 4bd7806b96..a1abe00ea4 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
@@ -51,6 +51,7 @@ Ops User Create
     Log  Output, govc role.usage: ${out}
 
 Run privilege-dependent docker operations
+    [Timeout]  15 minutes
     # Run containers with volumes and container networks to test scenarios requiring containerVMs
     # to have the highest privileges.
     ${rc}  ${output}=  Run And Return Rc And Output  docker %{VCH-PARAMS} pull ${busybox}
diff --git a/tests/resources/Docker-Util.robot b/tests/resources/Docker-Util.robot
index 63cf0012ee..e791d4ff67 100644
--- a/tests/resources/Docker-Util.robot
+++ b/tests/resources/Docker-Util.robot
@@ -123,6 +123,7 @@ Verify Container Rename
     Should Contain  ${output}  ${vmName}
 
 Run Regression Tests
+    [Timeout]  15 minutes
     ${rc}  ${output}=  Run And Return Rc And Output  docker %{VCH-PARAMS} pull ${busybox}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0

From 22eac05bb0056a6b9510c45edd6f838bd3f84d52 Mon Sep 17 00:00:00 2001
From: Anchal Agrawal <anchala@vmware.com>
Date: Thu, 3 May 2018 17:31:41 -0500
Subject: [PATCH 13/31] Fix tar extraction path in DRS-disabled nightlies
 (#7889) (#8151)

This change fixes the VIC tar extraction path in the
jenkins-drs-disabled-run.sh script. We were erroneously
extracting the binaries outside the vic/ directory, causing the
'Install VIC Appliance To Test Server' keyword to fail because it
couldn't locate the vic-machine binary in vic/bin.

Towards #7662

(cherry picked from commit 0cf0634297e09bf3d3175130e02c59e9f536672d)
---
 tests/drs-disabled/jenkins-drs-disabled-run.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/drs-disabled/jenkins-drs-disabled-run.sh b/tests/drs-disabled/jenkins-drs-disabled-run.sh
index 442fbe3975..491d6332ae 100755
--- a/tests/drs-disabled/jenkins-drs-disabled-run.sh
+++ b/tests/drs-disabled/jenkins-drs-disabled-run.sh
@@ -35,10 +35,10 @@ n=0
 n=0
    until [ $n -ge 5 ]
    do
-      mkdir bin
+      mkdir vic/bin
       echo "Extracting .tar.gz"
-      tar xvzf $input -C bin/ --strip 1
-      if [ -f "bin/vic-machine-linux" ]
+      tar xvzf $input -C vic/bin --strip 1
+      if [ -f "vic/bin/vic-machine-linux" ]
       then
       echo "tar extraction complete.."
       canContinue="Yes"
@@ -46,7 +46,7 @@ n=0
       else
       echo "tar extraction failed"
       canContinue="No"
-      rm -rf bin
+      rm -rf vic/bin
       fi
       n=$[$n+1]
       sleep 15
@@ -77,5 +77,5 @@ sed -i -e 's|FAIL|<font color="red">FAIL</font>|g' console.log
 
 # Run the log upload script in a container
 cmd="tests/drs-disabled/upload-logs.sh"
-docker run --rm -v $PWD/vic:/go --env-file $envfile $image $cmd
+docker run --rm -e BUILD_TIMESTAMP -v $PWD/vic:/go --env-file $envfile $image $cmd
 

From b9180ea2e7cdf683e7ceec35e295008be4766dee Mon Sep 17 00:00:00 2001
From: Anchal Agrawal <anchala@vmware.com>
Date: Fri, 4 May 2018 12:00:07 -0500
Subject: [PATCH 14/31] Add test for no-DRS VCH create with RP options (#7881)
 (#8151)

This commit adds an integration test in Group19-DRS-Disabled to
verify that during a VCH create, vic-machine checks for the DRS
setting in the targeted cluster and also warns if any command
options specific to resource pools are supplied, since they will
be ignored as they are not applicable in a DRS-disabled env.

Towards #7275

(cherry picked from commit e3991c1c7ebd645bd7cd6d2de191553ece56d525)
---
 .../19-4-DRS-Disabled-License-Features.md     | 19 +++++-------
 .../19-4-DRS-Disabled-License-Features.robot  | 31 +++++++++++++++++++
 2 files changed, 38 insertions(+), 12 deletions(-)
 create mode 100644 tests/manual-test-cases/Group19-DRS-Disabled/19-4-DRS-Disabled-License-Features.robot

diff --git a/tests/manual-test-cases/Group19-DRS-Disabled/19-4-DRS-Disabled-License-Features.md b/tests/manual-test-cases/Group19-DRS-Disabled/19-4-DRS-Disabled-License-Features.md
index 2167352e5e..824e7055bb 100644
--- a/tests/manual-test-cases/Group19-DRS-Disabled/19-4-DRS-Disabled-License-Features.md
+++ b/tests/manual-test-cases/Group19-DRS-Disabled/19-4-DRS-Disabled-License-Features.md
@@ -2,31 +2,26 @@ Test 19-4 - DRS Disabled License Features
 =======
 
 # Purpose:
-To verify that the license and feature checks required for a ROBO Advanced environment are displayed and updated on VCH Admin.
+To verify that vic-machine create checks for the DRS setting and command flags specific to resource pools in a vSphere environment without DRS, such as a ROBO deployment. If a VCH is being installed in a DRS-disabled environment, vic-machine create should warn that DRS is disabled and mention that any resource pool options supplied in the command will be ignored as they are not applicable in this environment.
 
 # References:
 1. [vSphere Remote Office and Branch Office](http://www.vmware.com/products/vsphere/remote-office-branch-office.html)
 2. [Provide License and Feature Check](https://github.com/vmware/vic/issues/7277)
-3. [vic-admin to report on license and feature compliance](https://github.com/vmware/vic/issues/7276)
+3. [vic-machine to provide license and feature check](https://github.com/vmware/vic/issues/7275)
 
 # Environment:
-This test requires access to VMware Nimbus cluster for dynamic ESXi and vCenter creation. This test should be executed in the following topologies and should have vSAN enabled.
-* 1 vCenter host with 3 clusters, where 1 cluster has 1 ESXi host and the other 2 clusters have 3 ESXi hosts each
-* 2 vCenter hosts connected with ELM, where each vCenter host has a cluster/host/datacenter topology that emulates a customer environment (exact topology TBD)
+This test requires access to VMware Nimbus cluster for dynamic ESXi and vCenter creation. This test should be executed in a vCenter environment with a cluster that has DRS turned off.
 
 See https://confluence.eng.vmware.com/display/CNA/VIC+ROBO for more details.
 
 # Test Steps:
-1. Deploy a ROBO Advanced vCenter testbed for both environments above
-2. Install a VCH on vCenter
-3. Visit the VCH Admin page and verify that the License and Feature Status sections show that required license and features are present
-4. Assign a more restrictive license such as ROBO Standard or Standard that does not have the required features (VDS, VSPC) to vCenter
-5. Assign the above license to each of the hosts within the vCenter cluster
-6. Refresh the VCH Admin page and verify that the License and Feature Status sections show that required license and features are not present
-7. Delete the VCH
+1. Deploy a vCenter testbed with DRS disabled
+2. Using vic-machine create, install a VCH with compute resource set to the cluster where DRS is off. Also supply options that are specific to resource pools: cpu, cpu-shares, cpu-reservation, memory, memory-shares and memory-reservation.
+3. Delete the VCH
 
 # Expected Outcome:
 * All test steps should complete without error
+* Step 2's output should contain a message stating that DRS is disabed and that the provided resource pool options will be ignored.
 
 # Possible Problems:
 None
diff --git a/tests/manual-test-cases/Group19-DRS-Disabled/19-4-DRS-Disabled-License-Features.robot b/tests/manual-test-cases/Group19-DRS-Disabled/19-4-DRS-Disabled-License-Features.robot
new file mode 100644
index 0000000000..2154d06d81
--- /dev/null
+++ b/tests/manual-test-cases/Group19-DRS-Disabled/19-4-DRS-Disabled-License-Features.robot
@@ -0,0 +1,31 @@
+# Copyright 2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+*** Settings ***
+Documentation  Test 19-4 - DRS-Disabled-License-Features
+Resource  ../../resources/Util.robot
+Suite Teardown  Cleanup VIC Appliance On Test Server
+
+*** Test Cases ***
+vic-machine create checks DRS setting and RP options
+    Set Environment Variable  TEST_TIMEOUT  30m
+
+    ${output}=  Install VIC Appliance To Test Server  additional-args=--memory 8000 --memory-reservation 512 --memory-shares 6000 --cpu 10000 --cpu-reservation 512 --cpu-shares high
+    Should Contain  ${output}  DRS is recommended, but is disabled
+    Should Contain  ${output}  Memory Limit
+    Should Contain  ${output}  Memory Reservation
+    Should Contain  ${output}  Memory Shares
+    Should Contain  ${output}  CPU Limit
+    Should Contain  ${output}  CPU Reservation
+    Should Contain  ${output}  CPU Shares

From 246796fe329829e568fc43a1c8b3d74fe132e236 Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Fri, 4 May 2018 13:40:32 -0500
Subject: [PATCH 15/31] New testbed for multiple cluster test (#7888) (#8151)

(cherry picked from commit 0ce740b0ecc37e26793b882a772d2663ce75698a)
---
 .../5-11-Multiple-Cluster.robot               |  64 +++-----
 tests/resources/Nimbus-Util.robot             |   6 +
 .../nimbus-testbeds/vic-multiple-cluster.rb   | 152 ++++++++++++++++++
 3 files changed, 183 insertions(+), 39 deletions(-)
 create mode 100644 tests/resources/nimbus-testbeds/vic-multiple-cluster.rb

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
index 92b50d1ec6..7fef7a00cb 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
@@ -16,7 +16,7 @@
 Documentation  Test 5-11 - Multiple Clusters
 Resource  ../../resources/Util.robot
 Suite Setup  Wait Until Keyword Succeeds  10x  10m  Multiple Cluster Setup
-Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
+Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-11-multiple-cluster*'
 
 *** Keywords ***
 # Insert elements from dict2 into dict1, overwriting conflicts in dict1 & returning new dict
@@ -30,51 +30,37 @@ Combine Dictionaries
 
 Multiple Cluster Setup
     [Timeout]    110 minutes
-    Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
-    &{esxes}=  Create Dictionary
-    ${num_of_esxes}=  Evaluate  2
-    :FOR  ${i}  IN RANGE  3
-    # Deploy some ESXi instances
-    \    &{new_esxes}=  Deploy Multiple Nimbus ESXi Servers in Parallel  ${num_of_esxes}  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
-    \    ${esxes}=  Combine Dictionaries  ${esxes}  ${new_esxes}
+    Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-11-multiple-cluster*'  ${false}
+    Log To Console  \nStarting testbed deploy...
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-multiple-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-multiple-cluster.rb --runName 5-11-multiple-cluster
+    Log  ${out}
 
-    # Investigate to see how many were actually deployed
-    \    ${len}=  Get Length  ${esxes}
-    \    ${num_of_esxes}=  Evaluate  ${num_of_esxes} - ${len}
-
-    # Exit if we've got enough & continue loop if we don't
-    \    Exit For Loop If  ${len} >= 2
-    \    Log To Console  Only got ${len} ESXi instance(s); Trying again
-
-    @{esx-names}=  Get Dictionary Keys  ${esxes}
-    @{esx-ips}=  Get Dictionary Values  ${esxes}
-    ${esx1}=  Get From List  ${esx-names}  0
-    ${esx2}=  Get From List  ${esx-names}  1
-    ${esx1-ip}=  Get From List  ${esx-ips}  0
-    ${esx2-ip}=  Get From List  ${esx-ips}  1
-
-    ${esx3}  ${esx4}  ${esx5}  ${vc}  ${esx3-ip}  ${esx4-ip}  ${esx5-ip}  ${vc-ip}=  Create a Simple VC Cluster  datacenter1  cls1
-    Set Suite Variable  @{list}  ${esx1}  ${esx2}  ${esx3}  ${esx4}  ${esx5}  %{NIMBUS_USER}-${vc}
-
-    Log To Console  Create cluster2 on the VC
-    ${out}=  Run  govc cluster.create cls2
-    Should Be Empty  ${out}
-    ${out}=  Run  govc cluster.add -hostname=${esx1-ip} -username=root -dc=datacenter1 -cluster=cls2 -password=e2eFunctionalTest -noverify=true
-    Should Contain  ${out}  OK
+    Open Connection  %{NIMBUS_GW}
+    Wait Until Keyword Succeeds  10 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
+    ${vc-ip}=  Get IP  5-11-multiple-cluster.vc.0
+    Close Connection
+    
+    Log To Console  Set environment variables up for GOVC
+    Set Environment Variable  GOVC_URL  ${vc-ip}
+    Set Environment Variable  GOVC_USERNAME  Administrator@vsphere.local
+    Set Environment Variable  GOVC_PASSWORD  Admin\!23
 
-    Log To Console  Create cluster3 on the VC
-    ${out}=  Run  govc cluster.create cls3
-    Should Be Empty  ${out}
-    ${out}=  Run  govc cluster.add -hostname=${esx2-ip} -username=root -dc=datacenter1 -cluster=cls3 -password=e2eFunctionalTest -noverify=true
-    Should Contain  ${out}  OK
+    Log To Console  Deploy VIC to the VC cluster
+    Set Environment Variable  TEST_URL_ARRAY  ${vc-ip}
+    Set Environment Variable  TEST_USERNAME  Administrator@vsphere.local
+    Set Environment Variable  TEST_PASSWORD  Admin\!23
+    Set Environment Variable  BRIDGE_NETWORK  bridge
+    Set Environment Variable  PUBLIC_NETWORK  vm-network
+    Remove Environment Variable  TEST_DATACENTER
+    Set Environment Variable  TEST_DATASTORE  local-0
+    Set Environment Variable  TEST_RESOURCE  cls
+    Set Environment Variable  TEST_TIMEOUT  15m
 
 *** Test Cases ***
 Test
     Log To Console  \nStarting test...
-    
+    Custom Testbed Keepalive  /dbc/pa-dbc1111/mhagen
 
     Install VIC Appliance To Test Server  certs=${false}  vol=default
-
     Run Regression Tests
-
     Cleanup VIC Appliance On Test Server
diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index 248580381b..b82c61be4d 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -230,6 +230,12 @@ Nimbus Cleanup
     ${list}=  Catenate  @{vm_list}
     Run Keyword And Ignore Error  Kill Nimbus Server  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  ${list}
 
+Nimbus Cleanup Single VM
+    [Arguments]  ${vm}  ${collect_log}=True  ${dontDelete}=${false}
+    Run Keyword If  ${collect_log}  Run Keyword And Continue On Failure  Gather Logs From Test Server
+    Return From Keyword If  ${dontDelete}
+    Run Keyword And Ignore Error  Kill Nimbus Server  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  ${vm}
+
 Gather Host IPs
     ${out}=  Run  govc ls host/cls
     ${out}=  Split To Lines  ${out}
diff --git a/tests/resources/nimbus-testbeds/vic-multiple-cluster.rb b/tests/resources/nimbus-testbeds/vic-multiple-cluster.rb
new file mode 100644
index 0000000000..2005eb13ee
--- /dev/null
+++ b/tests/resources/nimbus-testbeds/vic-multiple-cluster.rb
@@ -0,0 +1,152 @@
+oneGB = 1 * 1000 * 1000 # in KB
+ 
+$testbed = Proc.new do
+  {
+    "name" => "multiple-cluster",
+    "version" => 3,
+    "esx" => [ 
+        {
+            "name" => "esx.0",
+            "vc" => "vc.0",
+            "style" => "fullInstall",
+            "desiredPassword" => "e2eFunctionalTest",
+            "memory" => 8192, # 2x default
+            "disks" => [ 30 * oneGB, 30 * oneGB, 30 * oneGB],
+            "nics" => 2,
+            "clusterName" => "cls",
+        },
+        {
+            "name" => "esx.1",
+            "vc" => "vc.0",
+            "style" => "fullInstall",
+            "desiredPassword" => "e2eFunctionalTest",
+            "memory" => 8192, # 2x default
+            "disks" => [ 30 * oneGB, 30 * oneGB, 30 * oneGB],
+            "nics" => 2,
+            "clusterName" => "cls",
+        },
+        {
+            "name" => "esx.2",
+            "vc" => "vc.0",
+            "style" => "fullInstall",
+            "desiredPassword" => "e2eFunctionalTest",
+            "memory" => 8192, # 2x default
+            "disks" => [ 30 * oneGB, 30 * oneGB, 30 * oneGB],
+            "nics" => 2,
+            "clusterName" => "cls2",
+        },
+        {
+            "name" => "esx.3",
+            "vc" => "vc.0",
+            "style" => "fullInstall",
+            "desiredPassword" => "e2eFunctionalTest",
+            "memory" => 8192, # 2x default
+            "disks" => [ 30 * oneGB, 30 * oneGB, 30 * oneGB],
+            "nics" => 2,
+            "clusterName" => "cls3",
+        },
+        {
+            "name" => "esx.4",
+            "vc" => "vc.0",
+            "style" => "fullInstall",
+            "desiredPassword" => "e2eFunctionalTest",
+            "memory" => 8192, # 2x default
+            "disks" => [ 30 * oneGB, 30 * oneGB, 30 * oneGB],
+            "nics" => 2
+        }
+    ],
+ 
+    "vcs" => [
+      {
+        "name" => "vc.0",
+        "type" => "vcva",
+        "dcName" => "dc1",
+        "clusters" => [
+                       {"name" => "cls", "vsan" => false, "enableDrs" => true, "enableHA" => true},
+                       {"name" => "cls2", "vsan" => false, "enableDrs" => true, "enableHA" => true},
+                       {"name" => "cls3", "vsan" => false, "enableDrs" => true, "enableHA" => true}
+                      ],
+        "addHosts" => "multiple-clusters",
+      }
+    ],
+
+    "postBoot" => Proc.new do |runId, testbedSpec, vmList, catApi, logDir|
+      esxList = vmList['esx']
+        esxList.each do |host|
+          host.ssh do |ssh|
+            ssh.exec!("esxcli network firewall set -e false")
+          end
+      end
+      vc = vmList['vc'][0]
+      vim = VIM.connect vc.rbvmomiConnectSpec
+      datacenters = vim.serviceInstance.content.rootFolder.childEntity.grep(RbVmomi::VIM::Datacenter)
+      raise "Couldn't find a Datacenter precreated"  if datacenters.length == 0
+      datacenter = datacenters.first
+      Log.info "Found a datacenter successfully in the system, name: #{datacenter.name}"
+      clusters = datacenter.hostFolder.children
+      raise "Couldn't find a cluster precreated"  if clusters.length == 0
+      cluster = datacenter.hostFolder.allChildren.find{|x| x.name == 'cls'}
+      Log.info "Found a cluster successfully in the system, name: #{cluster.name}"
+
+      dvs = datacenter.networkFolder.CreateDVS_Task(
+        :spec => {
+          :configSpec => {
+            :name => "test-ds"
+          },
+	    }
+      ).wait_for_completion
+      Log.info "Vds DSwitch created"
+
+      dvpg1 = dvs.AddDVPortgroup_Task(
+        :spec => [
+          {
+            :name => "management",
+            :type => :earlyBinding,
+            :numPorts => 12,
+          }
+        ]
+      ).wait_for_completion
+      Log.info "management DPG created"
+
+      dvpg2 = dvs.AddDVPortgroup_Task(
+        :spec => [
+          {
+            :name => "vm-network",
+            :type => :earlyBinding,
+            :numPorts => 12,
+          }
+        ]
+      ).wait_for_completion
+      Log.info "vm-network DPG created"
+
+      dvpg3 = dvs.AddDVPortgroup_Task(
+        :spec => [
+          {
+            :name => "bridge",
+            :type => :earlyBinding,
+            :numPorts => 12,
+          }
+        ]
+      ).wait_for_completion
+      Log.info "bridge DPG created"
+
+      Log.info "Add hosts to the DVS"
+      onecluster_pnic_spec = [ VIM::DistributedVirtualSwitchHostMemberPnicSpec({:pnicDevice => 'vmnic1'}) ]
+      dvs_config = VIM::DVSConfigSpec({
+        :configVersion => dvs.config.configVersion,
+        :host => cluster.host.map do |host|
+        {
+          :operation => :add,
+          :host => host,
+          :backing => VIM::DistributedVirtualSwitchHostMemberPnicBacking({
+            :pnicSpec => onecluster_pnic_spec
+          })
+        }
+        end
+      })
+      dvs.ReconfigureDvs_Task(:spec => dvs_config).wait_for_completion
+      Log.info "Hosts added to DVS successfully"
+    end
+  }
+end
+

From d887fc9f1164bd69820ca7294179dc592f5c274b Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Fri, 4 May 2018 14:21:16 -0500
Subject: [PATCH 16/31] Switch to testbed for NFS Datastore test (#7900)
 (#8151)

(cherry picked from commit 29f07bb46f1b9251d39ca936ebcdfdadc27c6ee9)
---
 .../5-15-NFS-Datastore.robot                  | 38 +++++++++++++------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot
index 466ae5194f..10e3fa8414 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot
@@ -16,22 +16,36 @@
 Documentation  Test 5-15 - NFS Datastore
 Resource  ../../resources/Util.robot
 Suite Setup  Wait Until Keyword Succeeds  10x  10m  NFS Datastore Setup
-Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
+Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-15-nfs-datastore*'
 
 *** Keywords ***
 NFS Datastore Setup
     [Timeout]    110 minutes
-    Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
-    ${esx1}  ${esx2}  ${esx3}  ${vc}  ${esx1-ip}  ${esx2-ip}  ${esx3-ip}  ${vc-ip}=  Create a Simple VC Cluster  datacenter1  cls1
-    Set Suite Variable  @{list}  ${esx1}  ${esx2}  ${esx3}  %{NIMBUS_USER}-${vc}
-
-    ${name}  ${ip}=  Deploy Nimbus NFS Datastore  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
-    Append To List  ${list}  ${name}
-
-    ${out}=  Run  govc datastore.create -mode readWrite -type nfs -name nfsDatastore -remote-host ${ip} -remote-path /store /datacenter1/host/cls1
-    Should Be Empty  ${out}
-
-    Set Environment Variable  TEST_DATASTORE  nfsDatastore
+    Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-15-nfs-datastore*'  ${false}
+    Log To Console  \nStarting testbed deploy...    
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-simple-cluster.rb --runName 5-15-nfs-datastore
+    Log  ${out}
+
+    Open Connection  %{NIMBUS_GW}
+    Wait Until Keyword Succeeds  10 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
+    ${vc-ip}=  Get IP  5-15-nfs-datastore.vc.0
+    Close Connection
+
+    Log To Console  Set environment variables up for GOVC
+    Set Environment Variable  GOVC_URL  ${vc-ip}
+    Set Environment Variable  GOVC_USERNAME  Administrator@vsphere.local
+    Set Environment Variable  GOVC_PASSWORD  Admin\!23
+
+    Log To Console  Deploy VIC to the VC cluster
+    Set Environment Variable  TEST_URL_ARRAY  ${vc-ip}
+    Set Environment Variable  TEST_USERNAME  Administrator@vsphere.local
+    Set Environment Variable  TEST_PASSWORD  Admin\!23
+    Set Environment Variable  BRIDGE_NETWORK  bridge
+    Set Environment Variable  PUBLIC_NETWORK  vm-network
+    Remove Environment Variable  TEST_DATACENTER
+    Set Environment Variable  TEST_DATASTORE  nfs0-1
+    Set Environment Variable  TEST_RESOURCE  cls
+    Set Environment Variable  TEST_TIMEOUT  15m
 
 *** Test Cases ***
 Test

From 0cd4c2fb617d45011e086a7c42c652d441a2b5b9 Mon Sep 17 00:00:00 2001
From: Clint Greenwood <cgreenwood@vmware.com>
Date: Fri, 4 May 2018 16:48:11 -0400
Subject: [PATCH 17/31] VCH install timeout set to 30m for 5-21-Datastore-Path
 (#7902) (#8151)

Increases timeout from default 3m to 30m

Toward #7718

(cherry picked from commit 0e43ea63204d435bf6c356065da06fbbc3c4ee23)
---
 .../Group5-Functional-Tests/5-21-Datastore-Path.robot       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-21-Datastore-Path.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-21-Datastore-Path.robot
index 32bf8bf5a0..d2b4da24a1 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-21-Datastore-Path.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-21-Datastore-Path.robot
@@ -1,4 +1,4 @@
-# Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+# Copyright 2016-2018 VMware, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -57,7 +57,7 @@ Datastore - DS Scheme Specified in Volume Store
 
     Log To Console  \nRunning custom vic-machine create - with DS Scheme in Volume Store
     # Need to run custom vic-machine create to specify volume store with DS scheme
-    ${output}=  Run  bin/vic-machine-linux create --debug 1 --name=%{VCH-NAME} --target=%{TEST_URL_ARRAY} --user=%{TEST_USERNAME} --bridge-network=%{BRIDGE_NETWORK} --public-network=%{PUBLIC_NETWORK} --image-store=%{TEST_DATASTORE} --volume-store=${dsScheme}%{TEST_DATASTORE}/images:default --password=%{TEST_PASSWORD} --appliance-iso=bin/appliance.iso --bootstrap-iso=bin/bootstrap.iso --insecure-registry harbor.ci.drone.local --force --kv
+    ${output}=  Run  bin/vic-machine-linux create --debug 1 --name=%{VCH-NAME} --target=%{TEST_URL_ARRAY} --user=%{TEST_USERNAME} --bridge-network=%{BRIDGE_NETWORK} --public-network=%{PUBLIC_NETWORK} --image-store=%{TEST_DATASTORE} --volume-store=${dsScheme}%{TEST_DATASTORE}/images:default --password=%{TEST_PASSWORD} --appliance-iso=bin/appliance.iso --bootstrap-iso=bin/bootstrap.iso --insecure-registry harbor.ci.drone.local --force --kv --timeout=%{TEST_TIMEOUT}
     Should Contain  ${output}  Installer completed successfully
 
 
@@ -89,5 +89,5 @@ Datastore - Space in Path with Scheme
 
     Log To Console  \nRunning custom vic-machine create - with DS Scheme in Volume Store with space in path
     # Need to run custom vic-machine create to specify volume store with DS scheme
-    ${output}=  Run  bin/vic-machine-linux create --debug 1 --name=%{VCH-NAME} --target=%{TEST_URL_ARRAY} --user=%{TEST_USERNAME} --bridge-network=%{BRIDGE_NETWORK} --public-network=%{PUBLIC_NETWORK} --image-store=%{TEST_DATASTORE} --volume-store=${dsScheme}%{TEST_DATASTORE}/images:default --password=%{TEST_PASSWORD} --appliance-iso=bin/appliance.iso --bootstrap-iso=bin/bootstrap.iso --insecure-registry harbor.ci.drone.local --force --kv
+    ${output}=  Run  bin/vic-machine-linux create --debug 1 --name=%{VCH-NAME} --target=%{TEST_URL_ARRAY} --user=%{TEST_USERNAME} --bridge-network=%{BRIDGE_NETWORK} --public-network=%{PUBLIC_NETWORK} --image-store=%{TEST_DATASTORE} --volume-store=${dsScheme}%{TEST_DATASTORE}/images:default --password=%{TEST_PASSWORD} --appliance-iso=bin/appliance.iso --bootstrap-iso=bin/bootstrap.iso --insecure-registry harbor.ci.drone.local --force --kv --timeout=%{TEST_TIMEOUT}
     Should Contain  ${output}  Installer completed successfully

From 90998728939497c312d0afe2496cd58354452380 Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Mon, 7 May 2018 12:18:23 -0500
Subject: [PATCH 18/31] Add NFS datastore keyword back (#7906) (#8151)

(cherry picked from commit b7784b866eb63811ac9c453fbe1a8843398365a6)
---
 tests/resources/Nimbus-Util.robot | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index b82c61be4d..ec6bc7e216 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -433,6 +433,29 @@ Deploy Simple NFS Testbed
 
     [Return]  ${user}-${name}.nfs.0  ${user}-${name}.nfs.1  ${user}-${name}.esx.0  ${nfs-ip}  ${nfs-ro-ip}  ${esx-ip}
 
+Deploy Nimbus NFS Datastore
+    [Arguments]  ${user}  ${password}  ${additional-args}=
+    ${name}=  Evaluate  'NFS-' + str(random.randint(1000,9999)) + str(time.clock())  modules=random,time
+    Log To Console  \nDeploying Nimbus NFS server: ${name}
+    Open Connection  %{NIMBUS_GW}
+    Wait Until Keyword Succeeds  2 min  30 sec  Login  ${user}  ${password}
+
+    ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-nfsdeploy ${name} ${additional-args}
+    Log  ${out}
+    # Make sure the deploy actually worked
+    Should Contain  ${out}  To manage this VM use
+    # Now grab the IP address and return the name and ip for later use
+    @{out}=  Split To Lines  ${out}
+    :FOR  ${item}  IN  @{out}
+    \   ${status}  ${message}=  Run Keyword And Ignore Error  Should Contain  ${item}  IP is
+    \   Run Keyword If  '${status}' == 'PASS'  Set Suite Variable  ${line}  ${item}
+    @{gotIP}=  Split String  ${line}  ${SPACE}
+    ${ip}=  Remove String  @{gotIP}[5]  ,
+
+    Log To Console  Successfully deployed new NFS server - ${user}-${name}
+    Close connection
+    [Return]  ${user}-${name}  ${ip}
+
 Change ESXi Server Password
     [Arguments]  ${password}
     ${out}=  Run  govc host.account.update -id root -password ${password}

From b904d1b8c5220e8b31c86a1eea39a2ad35114ed3 Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Tue, 8 May 2018 11:06:05 -0500
Subject: [PATCH 19/31] Select correct datastore for the test (#7909) (#8151)

(cherry picked from commit 86c0c52bd6783508a4b060965bffb427a0b9c2ca)
---
 .../5-11-Multiple-Cluster.robot               | 22 ++++++++-----------
 tests/resources/Nimbus-Util.robot             |  6 +++++
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
index 7fef7a00cb..6eeebc80c2 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
@@ -16,18 +16,10 @@
 Documentation  Test 5-11 - Multiple Clusters
 Resource  ../../resources/Util.robot
 Suite Setup  Wait Until Keyword Succeeds  10x  10m  Multiple Cluster Setup
-Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-11-multiple-cluster*'
+Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-11-multiple-cluster*'  ${false}
+Test Teardown  Cleanup VIC Appliance On Test Server
 
 *** Keywords ***
-# Insert elements from dict2 into dict1, overwriting conflicts in dict1 & returning new dict
-Combine Dictionaries
-    [Arguments]  ${dict1}  ${dict2}
-    ${dict2keys}=  Get Dictionary Keys  ${dict2}
-    :FOR  ${key}  IN  @{dict2keys}
-    \    ${elem}=  Get From Dictionary  ${dict2}  ${key}
-    \    Set To Dictionary  ${dict1}  ${key}  ${elem}
-    [Return]  ${dict1}
-
 Multiple Cluster Setup
     [Timeout]    110 minutes
     Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-11-multiple-cluster*'  ${false}
@@ -52,9 +44,14 @@ Multiple Cluster Setup
     Set Environment Variable  BRIDGE_NETWORK  bridge
     Set Environment Variable  PUBLIC_NETWORK  vm-network
     Remove Environment Variable  TEST_DATACENTER
-    Set Environment Variable  TEST_DATASTORE  local-0
-    Set Environment Variable  TEST_RESOURCE  cls
     Set Environment Variable  TEST_TIMEOUT  15m
+    Set Environment Variable  TEST_RESOURCE  cls
+
+    # Get one of the hosts in the cluster we want and make sure we use the correct local datastore
+    ${hosts}=  Run  govc ls -t HostSystem host/cls
+    @{hosts}=  Split To Lines  ${hosts}
+    ${datastore}=  Get Name of First Local Storage For Host  @{hosts}[0]
+    Set Environment Variable  TEST_DATASTORE  "${datastore}"
 
 *** Test Cases ***
 Test
@@ -63,4 +60,3 @@ Test
 
     Install VIC Appliance To Test Server  certs=${false}  vol=default
     Run Regression Tests
-    Cleanup VIC Appliance On Test Server
diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index ec6bc7e216..edb51f490d 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -562,3 +562,9 @@ Is Nimbus Location WDC
     ${status}=  Run Keyword And Return Status  Should Not Be Empty  ${out}
     Close Connection
     [Return]  ${status}
+
+Get Name of First Local Storage For Host
+    [Arguments]  ${host}
+    ${datastores}=  Run  govc host.info -host ${host} -json | jq -r '.HostSystems[].Config.FileSystemVolume.MountInfo[].Volume | select (.Type\=\="VMFS") | select (.Local\=\=true) | .Name'
+    @{datastores}=  Split To Lines  ${datastores}
+    [Return]  @{datastores}[0]

From 4dfcaa6b076a0ab21d99ebfc46a96a4129ec7980 Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Tue, 8 May 2018 13:52:02 -0500
Subject: [PATCH 20/31] Create a new testbed that includes a static IP address
 worker (#7918) (#8151)

(cherry picked from commit de441362b70cd63e7cc06ad8e28c161f6202620f)
---
 tests/resources/Nimbus-Util.robot             |  13 +-
 .../vic-simple-cluster-with-static.rb         | 126 ++++++++++++++++++
 2 files changed, 132 insertions(+), 7 deletions(-)
 create mode 100644 tests/resources/nimbus-testbeds/vic-simple-cluster-with-static.rb

diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index edb51f490d..0ffe35ad08 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -487,23 +487,22 @@ Create Simple VC Cluster With Static IP
     [Timeout]    110 minutes
     Set Suite Variable  ${NIMBUS_LOCATION}  NIMBUS_LOCATION=wdc
     Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
-    Log To Console  Create a new simple vc cluser with static ip support...
-    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-simple-cluster.rb --runName ${name}
+    Log To Console  Create a new simple vc cluster with static ip support...
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster-with-static --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-simple-cluster-with-static.rb --runName ${name}
     Log  ${out}
 
     Open Connection  %{NIMBUS_GW}
     Wait Until Keyword Succeeds  10 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
     ${vc-ip}=  Get IP  ${name}.vc.0
-    ${pod}=  Fetch POD  ${name}.vc.0
-    Set Suite Variable  ${NIMBUS_POD}  ${pod}
+    ${worker-ip}=  Get IP  ${name}.worker.0
     Close Connection
 
-    Set Suite Variable  @{list}  %{NIMBUS_USER}-${name}.esx.0  %{NIMBUS_USER}-${name}.esx.1  %{NIMBUS_USER}-${name}.esx.2  %{NIMBUS_USER}-${name}.nfs.0  %{NIMBUS_USER}-${name}.vc.0
-    Log To Console  Finished Creating Cluster ${name}
+    Set Suite Variable  @{list}  %{NIMBUS_USER}-${name}.esx.0  %{NIMBUS_USER}-${name}.esx.1  %{NIMBUS_USER}-${name}.esx.2  %{NIMBUS_USER}-${name}.nfs.0  %{NIMBUS_USER}-${name}.vc.0  %{NIMBUS_USER}-${name}.worker.0
+    Log To Console  Finished creating cluster ${name}
 
+    Set Environment Variable  STATIC_WORKER_IP  ${worker-ip}
     ${out}=  Get Static IP Address
     Set Suite Variable  ${static}  ${out}
-    Append To List  ${list}  %{STATIC_WORKER_NAME}
 
     Log To Console  Set environment variables up for GOVC
     Set Environment Variable  GOVC_URL  ${vc-ip}
diff --git a/tests/resources/nimbus-testbeds/vic-simple-cluster-with-static.rb b/tests/resources/nimbus-testbeds/vic-simple-cluster-with-static.rb
new file mode 100644
index 0000000000..e49b343c3a
--- /dev/null
+++ b/tests/resources/nimbus-testbeds/vic-simple-cluster-with-static.rb
@@ -0,0 +1,126 @@
+oneGB = 1 * 1000 * 1000 # in KB
+ 
+$testbed = Proc.new do
+  {
+    "name" => "vic-simple-cluster-with-static",
+    "version" => 3,
+    "esx" => (0..2).map do | idx |
+      {
+        "name" => "esx.#{idx}",
+        "vc" => "vc.0",
+        "style" => "fullInstall",
+        "desiredPassword" => "e2eFunctionalTest",
+        "memory" => 8192, # 2x default
+        "disks" => [ 30 * oneGB, 30 * oneGB, 30 * oneGB],
+        "nics" => 2,
+        "mountNfs" => ["nfs.0"],
+        "clusterName" => "cls",
+      }
+    end,
+
+    "nfs" => [
+      {
+        "name" => "nfs.0",
+        "type" => "NFS41"
+      }
+    ],
+ 
+    "vcs" => [
+      {
+        "name" => "vc.0",
+        "type" => "vcva",
+        "dcName" => "dc1",
+        "clusters" => [{"name" => "cls", "vsan" => false, "enableDrs" => true, "enableHA" => true}],
+        "addHosts" => "allInSameCluster",
+      }
+    ],
+
+    "worker" => [
+      {
+         "name" => "worker.0",
+         "enableStaticIpService" => true,
+      },
+     ],
+
+    "postBoot" => Proc.new do |runId, testbedSpec, vmList, catApi, logDir|
+      workerVM = vmList['worker'][0]
+      Log.info "static ip service endpoint is #{workerVM.info['nsips']}"
+      esxList = vmList['esx']
+        esxList.each do |host|
+          host.ssh do |ssh|
+            ssh.exec!("esxcli network firewall set -e false")
+          end
+      end
+      vc = vmList['vc'][0]
+      vim = VIM.connect vc.rbvmomiConnectSpec
+      datacenters = vim.serviceInstance.content.rootFolder.childEntity.grep(RbVmomi::VIM::Datacenter)
+      raise "Couldn't find a Datacenter precreated"  if datacenters.length == 0
+      datacenter = datacenters.first
+      Log.info "Found a datacenter successfully in the system, name: #{datacenter.name}"
+      clusters = datacenter.hostFolder.children
+      raise "Couldn't find a cluster precreated"  if clusters.length == 0
+      cluster = clusters.first
+      Log.info "Found a cluster successfully in the system, name: #{cluster.name}"
+
+      dvs = datacenter.networkFolder.CreateDVS_Task(
+        :spec => {
+          :configSpec => {
+            :name => "test-ds"
+          },
+	    }
+      ).wait_for_completion
+      Log.info "Vds DSwitch created"
+
+      dvpg1 = dvs.AddDVPortgroup_Task(
+        :spec => [
+          {
+            :name => "management",
+            :type => :earlyBinding,
+            :numPorts => 12,
+          }
+        ]
+      ).wait_for_completion
+      Log.info "management DPG created"
+
+      dvpg2 = dvs.AddDVPortgroup_Task(
+        :spec => [
+          {
+            :name => "vm-network",
+            :type => :earlyBinding,
+            :numPorts => 12,
+          }
+        ]
+      ).wait_for_completion
+      Log.info "vm-network DPG created"
+
+      dvpg3 = dvs.AddDVPortgroup_Task(
+        :spec => [
+          {
+            :name => "bridge",
+            :type => :earlyBinding,
+            :numPorts => 12,
+          }
+        ]
+      ).wait_for_completion
+      Log.info "bridge DPG created"
+
+      Log.info "Add hosts to the DVS"
+      onecluster_pnic_spec = [ VIM::DistributedVirtualSwitchHostMemberPnicSpec({:pnicDevice => 'vmnic1'}) ]
+      dvs_config = VIM::DVSConfigSpec({
+        :configVersion => dvs.config.configVersion,
+        :host => cluster.host.map do |host|
+        {
+          :operation => :add,
+          :host => host,
+          :backing => VIM::DistributedVirtualSwitchHostMemberPnicBacking({
+            :pnicSpec => onecluster_pnic_spec
+          })
+        }
+        end
+      })
+      dvs.ReconfigureDvs_Task(:spec => dvs_config).wait_for_completion
+      Log.info "Hosts added to DVS successfully"
+    end
+  }
+end
+

From e9836e208f25138bf4868d7b1f6174ba0b0e9765 Mon Sep 17 00:00:00 2001
From: Andrew Chin <andrewtchin@users.noreply.github.com>
Date: Fri, 11 May 2018 16:21:06 -0500
Subject: [PATCH 21/31] Add ping to NFS Volume nightly tests (#7821) (#8151)

(cherry picked from commit 71664a510d700f4ee829f1caf971bececd235405)
---
 .../5-22-NFS-Volume.robot                     | 30 ++++++++++++++++++-
 tests/resources/Network-Util.robot            | 25 ++++++++++++++++
 tests/resources/Util.robot                    |  1 +
 3 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 tests/resources/Network-Util.robot

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
index 43b7ca725e..5311f66209 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
@@ -60,7 +60,7 @@ Setup ESX And NFS Suite
     ${out}=  Execute Command   rpcdebug -m rpc -s all
     ${out}=  Execute Command   service rpcbind restart
     Close Connection
-    
+
     Open Connection  ${NFS_READONLY_IP}
     Wait Until Keyword Succeeds  2 min  30 sec  Login  root  ${strippedPW}
     ${out}=  Execute Command   rpcdebug -m nfsd -s all
@@ -86,6 +86,8 @@ Setup ENV Variables for VIC Appliance Install
 Verify NFS Volume Basic Setup
     [Arguments]  ${volumeName}  ${containerName}  ${nfsIP}  ${rwORro}
 
+    Wait For NFS And VCH
+
     ${rc}  ${output}=  Run And Return Rc And Output  docker %{VCH-PARAMS} run --name ${containerName} -v ${volumeName}:/mydata ${busybox} mount
     Should Be Equal As Integers  ${rc}  0
     Should Contain  ${output}  ${nfsIP}:/store/volumes/${volumeName}
@@ -100,6 +102,8 @@ Verify NFS Volume Basic Setup
 
 Verify NFS Volume Already Created
     [Arguments]  ${containerVolName}
+    Wait For NFS And VCH
+
     ${rc}  ${output}=  Run And Return Rc And Output  docker %{VCH-PARAMS} volume create --name=${containerVolName} --opt VolumeStore=${nfsVolumeStore}
     Should Be Equal As Integers  ${rc}  1
     Should Contain  ${output}  Error response from daemon: A volume named ${containerVolName} already exists. Choose a different volume name.
@@ -139,6 +143,10 @@ NFS Volume Cleanup
     Gather NFS Logs
     Nimbus Cleanup  ${list}
 
+Wait For NFS And VCH
+    Wait Until Keyword Succeeds  20x  30s  Ping Host Successfully  ${NFS_READONLY_IP}
+    Wait Until Keyword Succeeds  20x  30s  Ping Host Successfully  %{VCH-IP}
+
 *** Test Cases ***
 VIC Appliance Install with Read Only NFS Volume
     Setup ENV Variables for VIC Appliance Install
@@ -150,6 +158,8 @@ VIC Appliance Install with Read Only NFS Volume
     Should Contain  ${output}  VolumeStore (${nfsReadOnlyVolumeStore}) cannot be brought online - check network, nfs server, and --volume-store configurations
     Should Contain  ${output}  Not all configured volume stores are online - check port layer log via vicadmin
 
+    Wait For NFS And VCH
+
     ${rc}  ${volumeOutput}=  Run And Return Rc And Output  docker %{VCH-PARAMS} volume create --opt VolumeStore=${nfsReadOnlyVolumeStore}
     Should Be Equal As Integers  ${rc}  1
     Should Contain  ${volumeOutput}  Error response from daemon: No volume store named (${nfsReadOnlyVolumeStore}) exists
@@ -174,6 +184,8 @@ VIC Appliance Install With Correct NFS Server
 Simple Docker Volume Create
     #Pull image  ${busybox}
 
+    Wait For NFS And VCH
+
     ${rc}  ${volumeOutput}=  Run And Return Rc And Output  docker %{VCH-PARAMS} volume create --opt VolumeStore=${nfsVolumeStore}
     Should Be Equal As Integers  ${rc}  0
 
@@ -182,6 +194,8 @@ Simple Docker Volume Create
     Verify NFS Volume Basic Setup  ${nfsUnNamedVolume}  ${unnamedNFSVolContainer}  ${NFS_IP}  rw
 
 Docker Volume Create Named Volume
+    Wait For NFS And VCH
+
     ${rc}  ${volumeOutput}=  Run And Return Rc And Output  docker %{VCH-PARAMS} volume create --name nfs-volume_%{VCH-NAME} --opt VolumeStore=${nfsVolumeStore}
     Should Be Equal As Integers  ${rc}  0
     Should Be Equal As Strings  ${volumeOutput}  nfs-volume_%{VCH-NAME}
@@ -196,12 +210,16 @@ Docker Volume Create Already Named Volume
     Run Keyword And Ignore Error  Verify NFS Volume Already Created  ${nfsNamedVolume}
 
 Docker Volume Create with possibly Invalid Name
+    Wait For NFS And VCH
+
     ${rc}  ${output}=  Run And Return Rc And Output  docker %{VCH-PARAMS} volume create --name="test!@\#$%^&*()" --opt VolumeStore=${nfsVolumeStore}
     Should Be Equal As Integers  ${rc}  1
     Should Be Equal As Strings  ${output}  Error response from daemon: volume name "test!@\#$%^&*()" includes invalid characters, only "[a-zA-Z0-9][a-zA-Z0-9_.-]" are allowed
 
 Docker Single Write and Read to/from File from one Container using NFS Volume
     # Done with the same container for this test.
+    Wait For NFS And VCH
+
     ${rc}  ${output}=  Run And Return Rc And Output  docker %{VCH-PARAMS} run --name ${createFileContainer} -d -v ${nfsNamedVolume}:/mydata ${busybox} /bin/top -d 600
     Should Be Equal As Integers  ${rc}  0
 
@@ -217,6 +235,8 @@ Docker Single Write and Read to/from File from one Container using NFS Volume
     Should Contain  ${output}  The Texas and Chile flag look similar.
 
 Docker Multiple Writes from Multiple Containers (one at a time) and Read from Another
+    Wait For NFS And VCH
+
     ${rc}  ${output}=  Run And Return Rc And Output  docker %{VCH-PARAMS} run -v ${nfsNamedVolume}:/mydata ${busybox} sh -c "echo 'The Chad and Romania flag look the same.\n' >> /mydata/test_nfs_file.txt"
     Should Be Equal As Integers  ${rc}  0
 
@@ -234,6 +254,8 @@ Docker Multiple Writes from Multiple Containers (one at a time) and Read from An
     Should Contain  ${output}  Norway and Iceland have flags that are basically inverses of each other.
 
 Docker Read and Remove File
+    Wait For NFS And VCH
+
     ${rc}  ${catID}=  Run And Return Rc And Output  docker %{VCH-PARAMS} run -d -v ${nfsNamedVolume}:/mydata ${busybox} sh -c "cat mydata/test_nfs_file.txt"
     Should Be Equal As Integers  ${rc}  0
     ${rc}  ${output}=  Run And Return Rc And Output  docker %{VCH-PARAMS} logs ${catID}
@@ -276,6 +298,8 @@ Simultaneous Container Write to File
 
 
 Simple Docker Volume Inspect
+    Wait For NFS And VCH
+
     ${rc}  ${output}=  Run And Return Rc And Output  docker %{VCH-PARAMS} volume inspect ${nfsNamedVolume}
     Should Be Equal As Integers  ${rc}  0
     ${output}=  Evaluate  json.loads(r'''${output}''')  json
@@ -305,6 +329,8 @@ Volume rm Tests
     Should Contain  ${output}  Error response from daemon: volume ${nfsNamedVolume} in use by
 
 Docker Inspect Mount Data after Reboot
+    Wait For NFS And VCH
+
     ${rc}  ${container}=  Run And Return Rc And Output  docker %{VCH-PARAMS} create --name ${mntDataTestContainer} -v ${mntTest} -v ${nfsNamedVolume}:${mntNamed} ${busybox}
     Should Be Equal As Integers  ${rc}  0
 
@@ -321,6 +347,8 @@ Docker Inspect Mount Data after Reboot
     Verify Volume Inspect Info  After VM Reboot  ${mntDataTestContainer}  ${checkList}
 
 Kill NFS Server
+    Wait For NFS And VCH
+
     Sleep  5 minutes
     ${rc}  ${runningContainer}=  Run And Return Rc And Output  docker %{VCH-PARAMS} run -d -v ${nfsNamedVolume}:/mydata ${busybox} sh -c "while true; do echo 'Still here...\n' >> /mydata/test_nfs_kill.txt; sleep 2; done"
     Should Be Equal As Integers  ${rc}  0
diff --git a/tests/resources/Network-Util.robot b/tests/resources/Network-Util.robot
new file mode 100644
index 0000000000..ad64acc0df
--- /dev/null
+++ b/tests/resources/Network-Util.robot
@@ -0,0 +1,25 @@
+# Copyright 2018 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+*** Settings ***
+Documentation  This resource contains any keywords related to networking
+
+*** Keywords ***
+Ping Host Successfully
+    [Arguments]  ${host}
+    ${rc}  ${output}=  Run And Return Rc And Output  ping -c 1 ${host}
+    Log  ${output}
+    Should Be Equal As Integers  ${rc}  0
+    Should Contain  ${output}  1 packets transmitted, 1 packets received, 0% packet loss
+    [Return]  ${output}
diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot
index f4ce326fd3..c25b0d154e 100644
--- a/tests/resources/Util.robot
+++ b/tests/resources/Util.robot
@@ -32,6 +32,7 @@ Resource  Admiral-Util.robot
 Resource  OVA-Util.robot
 Resource  Cert-Util.robot
 Resource  Slack-Util.robot
+Resource  Network-Util.robot
 Resource  nimbus-testbeds/ELM-DRS-Disabled.robot
 
 Variables  dynamic-vars.py

From 41df3550bfb56aebec190c35d830452763320ead Mon Sep 17 00:00:00 2001
From: Mikkel Hagen <mhagen@vmware.com>
Date: Mon, 21 May 2018 16:25:44 -0500
Subject: [PATCH 22/31] Fix VSAN testbed for 6.7 testing (#7971) (#8151)

(cherry picked from commit 4573c6042773919a0bd72ecd2724158ebe5084e5)
---
 tests/nightly/jenkins-nightly-run.sh        | 2 +-
 tests/resources/nimbus-testbeds/vic-vsan.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh
index 8ac0c06b18..07f26b4e33 100755
--- a/tests/nightly/jenkins-nightly-run.sh
+++ b/tests/nightly/jenkins-nightly-run.sh
@@ -83,7 +83,7 @@ elif [[ $target == "6.5" ]]; then
     cat 65/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log
 elif [[ $target == "6.7" ]]; then
     echo "Executing nightly tests on vSphere 6.7"
-    pabot --processes 4 --removekeywords TAG:secret --exclude nsx --exclude hetero --exclude vsan-complex --variable ESX_VERSION:$ESX_67_VERSION --variable VC_VERSION:$VC_67_VERSION -d 67/$i tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests
+    pabot --processes 4 --removekeywords TAG:secret --exclude nsx --exclude hetero --variable ESX_VERSION:$ESX_67_VERSION --variable VC_VERSION:$VC_67_VERSION -d 67/$i tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests
     cat 67/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log
 fi
 # Pretty up the email results
diff --git a/tests/resources/nimbus-testbeds/vic-vsan.rb b/tests/resources/nimbus-testbeds/vic-vsan.rb
index bf5726659e..f934be0384 100644
--- a/tests/resources/nimbus-testbeds/vic-vsan.rb
+++ b/tests/resources/nimbus-testbeds/vic-vsan.rb
@@ -141,7 +141,7 @@
     testbed['vc']['dbHost'] = 'vc-mssql'
   end
 
-  testbed = VcQaTestbedCommon.addSharedDisks testbed, [20, 10, 20, 10], sharedStorageStyle   # 2 x 20gb shared vmfs, 2 x 10gb free luns as defined by 'freeSharedLuns', DON'T CHANGE THE ORDERING UNLESS YOU KNOW WHAT YOU'RE DOING!
+  testbed = VcQaTestbedCommon.addSharedDisks testbed, [10, 10, 20, 20], sharedStorageStyle   # 2 x 20gb shared vmfs, 2 x 10gb free luns as defined by 'freeSharedLuns', DON'T CHANGE THE ORDERING UNLESS YOU KNOW WHAT YOU'RE DOING!
 
   testbed
 end

From a50da2d8037f3586dcd0c61eff922d2a5f1314ea Mon Sep 17 00:00:00 2001
From: singhshwetaQE <singhshweta@vmware.com>
Date: Tue, 29 May 2018 13:32:15 -0500
Subject: [PATCH 23/31] Improving 21-1-Whitelist Test -Harbor Setup (#7997)
 (#8151)

* fix 21-1-Whitelist test

(cherry picked from commit ef61c13035a713d60c05e64ea542c7315cc4430f)
---
 .../manual-test-cases/Group21-Registries/21-1-Whitelist.robot | 1 +
 tests/resources/Harbor-Util.robot                             | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/manual-test-cases/Group21-Registries/21-1-Whitelist.robot b/tests/manual-test-cases/Group21-Registries/21-1-Whitelist.robot
index da714f460a..b7ce2bb732 100644
--- a/tests/manual-test-cases/Group21-Registries/21-1-Whitelist.robot
+++ b/tests/manual-test-cases/Group21-Registries/21-1-Whitelist.robot
@@ -42,6 +42,7 @@ Setup Harbor
     Simple ESXi Setup
 
     # Install a Harbor server with HTTPS a Harbor server with HTTP
+    Run Keyword and Ignore Error  Remove File  ${HARBOR_VERSION}.ova
     Install Harbor To Test Server  protocol=https  name=harbor-https
     Set Environment Variable  HTTPS_HARBOR_IP  %{HARBOR-IP}
 
diff --git a/tests/resources/Harbor-Util.robot b/tests/resources/Harbor-Util.robot
index d9e7d38b74..5946fb3889 100644
--- a/tests/resources/Harbor-Util.robot
+++ b/tests/resources/Harbor-Util.robot
@@ -36,7 +36,9 @@ Install Harbor To Test Server
     [Arguments]  ${name}=harbor  ${protocol}=http  ${verify}=off  ${host}=%{TEST_URL_ARRAY}  ${datastore}=%{TEST_DATASTORE}  ${network}=VM Network
     Log To Console  \nFetching harbor ova...
     ${status}  ${message}=  Run Keyword And Ignore Error  OperatingSystem.File Should Exist  ${HARBOR_VERSION}.ova
-    ${out}=  Run Keyword If  '${status}' == 'FAIL'  Run  wget https://github.com/vmware/harbor/releases/download/${HARBOR_SHORT_VERSION}/${HARBOR_VERSION}.ova
+    ${rc}  ${output}=  Run Keyword If  '${status}' == 'FAIL'  Run And Return Rc And Output  wget https://github.com/vmware/harbor/releases/download/${HARBOR_SHORT_VERSION}/${HARBOR_VERSION}.ova
+    Log  ${output}
+    Should Be Equal As Integers  ${rc}  0
     ${status}  ${message}=  Run Keyword And Ignore Error  Environment Variable Should Be Set  DRONE_BUILD_NUMBER
     Run Keyword If  '${status}' == 'FAIL'  Set Environment Variable  DRONE_BUILD_NUMBER  0
     @{URLs}=  Split String  %{TEST_URL_ARRAY}

From 450ec927ed2080b960125c8c7a89643a0a808ef9 Mon Sep 17 00:00:00 2001
From: singhshwetaQE <singhshweta@vmware.com>
Date: Wed, 30 May 2018 16:26:03 -0500
Subject: [PATCH 24/31] fix harbor-util (#8011) (#8151)

(cherry picked from commit 2c5ded5c76c2dfb15739ab7404eb649d8c236671)
---
 tests/resources/Harbor-Util.robot | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/resources/Harbor-Util.robot b/tests/resources/Harbor-Util.robot
index 5946fb3889..6000603c04 100644
--- a/tests/resources/Harbor-Util.robot
+++ b/tests/resources/Harbor-Util.robot
@@ -36,9 +36,7 @@ Install Harbor To Test Server
     [Arguments]  ${name}=harbor  ${protocol}=http  ${verify}=off  ${host}=%{TEST_URL_ARRAY}  ${datastore}=%{TEST_DATASTORE}  ${network}=VM Network
     Log To Console  \nFetching harbor ova...
     ${status}  ${message}=  Run Keyword And Ignore Error  OperatingSystem.File Should Exist  ${HARBOR_VERSION}.ova
-    ${rc}  ${output}=  Run Keyword If  '${status}' == 'FAIL'  Run And Return Rc And Output  wget https://github.com/vmware/harbor/releases/download/${HARBOR_SHORT_VERSION}/${HARBOR_VERSION}.ova
-    Log  ${output}
-    Should Be Equal As Integers  ${rc}  0
+    Run Keyword If  '${status}' == 'FAIL'  Download Harbor ova
     ${status}  ${message}=  Run Keyword And Ignore Error  Environment Variable Should Be Set  DRONE_BUILD_NUMBER
     Run Keyword If  '${status}' == 'FAIL'  Set Environment Variable  DRONE_BUILD_NUMBER  0
     @{URLs}=  Split String  %{TEST_URL_ARRAY}
@@ -89,6 +87,12 @@ Install Harbor Self Signed Cert
     ${out}=  Run  systemctl daemon-reload
     ${out}=  Run  systemctl restart docker
 
+Download Harbor ova
+    ${rc}  ${output}=   Run And Return Rc And Output  wget https://github.com/vmware/harbor/releases/download/${HARBOR_SHORT_VERSION}/${HARBOR_VERSION}.ova
+    Log  ${output}
+    Should Be Equal As Integers  ${rc}  0
+    Log To Console    Harbor OVA download successful
+
 Log Into Harbor
     [Arguments]  ${user}=%{TEST_USERNAME}  ${pw}=%{TEST_PASSWORD}
     Maximize Browser Window

From 4d141d1a7984b125b9d777b99d22626165268ad8 Mon Sep 17 00:00:00 2001
From: George Hicken <hickeng@users.noreply.github.com>
Date: Fri, 15 Jun 2018 16:35:03 -0700
Subject: [PATCH 25/31] Parameterize nightly job script (#8047) (#8151)

This parameterizes the nightly job runner script so that we can pass in
the configuration elements that were previously hard coded. The primary
difference is it's possible to run a user specified subset of the test
cases.

Also removes references in the test definitions to specific user
directories.

There is a mostly duplicated version of the job runner files called
jenkins-drs-disabled-run.sh - as best I can determine these files are not
unified only because some of the nightly tests have not been confirmed to
work when the test executor is in a container.

(cherry picked from commit 3b269fa70788a1212f2987df1bcf3668af54a21e)
---
 .../5-11-Multiple-Cluster.robot               |   3 +-
 .../5-15-NFS-Datastore.robot                  |   2 +-
 .../5-22-NFS-Volume.robot                     |   2 +-
 .../5-26-Static-IP-Address.robot              |   1 -
 .../5-27-Selenium-Grid.robot                  |   3 +-
 .../5-6-1-VSAN-Simple.robot                   |   2 -
 .../5-6-2-VSAN-Complex.robot                  |   2 -
 tests/nightly/jenkins-nightly-run.sh          | 140 ++++++++++--------
 tests/nightly/upload-logs.sh                  |  53 +++----
 tests/resources/Nimbus-Util.robot             |  46 +++---
 10 files changed, 135 insertions(+), 119 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
index 6eeebc80c2..24a90676c4 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
@@ -24,7 +24,7 @@ Multiple Cluster Setup
     [Timeout]    110 minutes
     Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-11-multiple-cluster*'  ${false}
     Log To Console  \nStarting testbed deploy...
-    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-multiple-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-multiple-cluster.rb --runName 5-11-multiple-cluster
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  spec=vic-multiple-cluster.rb  args=--noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-multiple-cluster --runName 5-11-multiple-cluster
     Log  ${out}
 
     Open Connection  %{NIMBUS_GW}
@@ -56,7 +56,6 @@ Multiple Cluster Setup
 *** Test Cases ***
 Test
     Log To Console  \nStarting test...
-    Custom Testbed Keepalive  /dbc/pa-dbc1111/mhagen
 
     Install VIC Appliance To Test Server  certs=${false}  vol=default
     Run Regression Tests
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot
index 10e3fa8414..3008e5f590 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot
@@ -23,7 +23,7 @@ NFS Datastore Setup
     [Timeout]    110 minutes
     Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-15-nfs-datastore*'  ${false}
     Log To Console  \nStarting testbed deploy...    
-    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-simple-cluster.rb --runName 5-15-nfs-datastore
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  spec=vic-simple-cluster.rb  args=--noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster --runName 5-15-nfs-datastore
     Log  ${out}
 
     Open Connection  %{NIMBUS_GW}
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
index 5311f66209..0590553911 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
@@ -38,7 +38,7 @@ Setup ESX And NFS Suite
     Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
     Log To Console  \nStarting test...
 
-    ${nfs}  ${nfs_ro}  ${esx1}  ${nfs_ip}  ${nfs_ro_ip}  ${esx1_ip}=  Deploy Simple NFS Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  additional-args=--testbedSpecRubyFile /dbc/w3-dbc302/rashok/vic-nfs.rb --esxBuild ${ESX_VERSION} --esxPxeDir ${ESX_VERSION} --plugin testng
+    ${nfs}  ${nfs_ro}  ${esx1}  ${nfs_ip}  ${nfs_ro_ip}  ${esx1_ip}=  Deploy Simple NFS Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  spec=vic-nfs.rb  args=--esxBuild ${ESX_VERSION} --esxPxeDir ${ESX_VERSION} --plugin testng
 
     Set Suite Variable  @{list}  ${esx1}  ${nfs}  ${nfs_ro}
     Set Suite Variable  ${ESX1}  ${esx1}
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot
index 65adb78c3f..937569d72f 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot
@@ -27,7 +27,6 @@ Setup VC With Static IP
 *** Test Cases ***
 Test
     Log To Console  \nStarting test...
-    Custom Testbed Keepalive  /dbc/pa-dbc1111/mhagen
 
     Install VIC Appliance To Test Server  additional-args=--public-network-ip &{static}[ip]/&{static}[netmask] --public-network-gateway &{static}[gateway] --dns-server 10.170.16.48
     Run Regression Tests
\ No newline at end of file
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot
index ac0fef639b..573baef0e7 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot
@@ -24,7 +24,7 @@ Selenium Grid Test Setup
     Log To Console  Starting testbed deployment...
     Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
     ${name}=  Evaluate  'vic-5-27-' + str(random.randint(1000,9999))  modules=random
-    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-iscsi-cluster --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-iscsi-cluster.rb --runName ${name}
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  spec=vic-iscsi-cluster.rb  args=--noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-iscsi-cluster --runName ${name}
     Log  ${out}
 
     Open Connection  %{NIMBUS_GW}
@@ -71,7 +71,6 @@ Wait Until Selenium Node Is Ready
 *** Test Cases ***    
 Test
     Log To Console  Starting Selenium Grid test...
-    Custom Testbed Keepalive  /dbc/pa-dbc1111/mhagen
 
     Install VIC Appliance To Test Server
 
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot
index 0efe749b44..abb4e6bd85 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot
@@ -68,8 +68,6 @@ Check VSAN DOMs In Datastore
 Simple VSAN
     Wait Until Keyword Succeeds  10x  30s  Check VSAN DOMs In Datastore  %{TEST_DATASTORE}
 
-    Custom Testbed Keepalive  /dbc/pa-dbc1111/mhagen
-
     Install VIC Appliance To Test Server
     Run Regression Tests
     Cleanup VIC Appliance On Test Server
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot
index 5197a7eefe..5c0306b646 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot
@@ -71,8 +71,6 @@ Check VSAN DOMs In Datastore
 Complex VSAN
     Wait Until Keyword Succeeds  10x  30s  Check VSAN DOMs In Datastore  %{TEST_DATASTORE}
 
-    Custom Testbed Keepalive  /dbc/pa-dbc1111/mhagen
-
     Install VIC Appliance To Test Server
     Run Regression Tests
     Cleanup VIC Appliance On Test Server
diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh
index 07f26b4e33..ec29aabfd8 100755
--- a/tests/nightly/jenkins-nightly-run.sh
+++ b/tests/nightly/jenkins-nightly-run.sh
@@ -16,82 +16,100 @@
 ESX_60_VERSION="ob-5251623"
 VC_60_VERSION="ob-5112509"
 
+ESX_65_VERSION="ob-7867845"
+VC_65_VERSION="ob-7867539"
+
 ESX_67_VERSION="ob-8169922"
 VC_67_VERSION="ob-8217866"
 
+DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs"
+DEFAULT_VCH_BUILD="*"
+DEFAULT_TESTCASES="tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests"
+
+
 if [[ $1 != "6.0" && $1 != "6.5" && $1 != "6.7" ]]; then
-    echo "Please specify a target cluster. One of: 6.0, 6.5, 6.7"
+    echo "Please specify a target version. One of: 6.0, 6.5, 6.7"
     exit 1
 fi
+
+# process the CLI arguments
 target="$1"
-echo "Target cluster: "$target
-
-input=$(gsutil ls -l gs://vic-engine-builds/vic_* | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | cut -d '/' -f 4)
-buildNumber=${input:4}
-
-n=0
-   until [ $n -ge 5 ]
-   do
-      echo "Retry.. $n"
-      echo "Downloading gcp file $input"
-      wget https://storage.googleapis.com/vic-engine-builds/$input
-      if [ -f "$input" ]
-      then
-      echo "File found.."
-      break
-      else
-      echo "File NOT found"
-      fi
-      n=$[$n+1]
-      sleep 15
-   done
-
-n=0
-   until [ $n -ge 5 ]
-   do
-      mkdir bin
-      echo "Extracting .tar.gz"
-      tar xvzf $input -C bin/ --strip 1
-      if [ -f "bin/vic-machine-linux" ]
-      then
-      echo "tar extraction complete.."
-      canContinue="Yes"
-      break
-      else
-      echo "tar extraction failed"
-      canContinue="No"
-      rm -rf bin
-      fi
-      n=$[$n+1]
-      sleep 15
-   done
-
-if [[ $canContinue = "No" ]]; then
+echo "Target version: ${target}"
+shift
+# Take the remaining CLI arguments as a test case list
+testcases="${*:-$DEFAULT_TESTCASES}"
+
+# TODO: the version downloaded by this logic is not coupled with the tests that will be run against it. This should be altered to pull a version that matches the commit SHA of the tests
+# we will be running or similar mechanism.
+VCH_BUILD=${VCH_BUILD:-${DEFAULT_VCH_BUILD}}
+input=$(gsutil ls -l gs://vic-engine-builds/vic_${VCH_BUILD} | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | cut -d '/' -f 4)
+VCH_BUILD=${input:4}
+
+# Enforce short SHA
+GIT_COMMIT=${GIT_COMMIT:0:7}
+
+case "$target" in
+    "6.0")
+        excludes="--exclude nsx"
+        ESX_BUILD=${ESX_BUILD:-$ESX_60_VERSION}
+        VC_BUILD=${VC_BUILD:-$VC_60_VERSION}
+        ;;
+    "6.5")
+        ESX_BUILD=${ESX_BUILD:-$ESX_65_VERSION}
+        VC_BUILD=${VC_BUILD:-$VC_65_VERSION}
+        ;;
+    "6.7")
+        excludes="--exclude nsx --exclude hetero"
+        ESX_BUILD=${ESX_BUILD:-$ESX_67_VERSION}
+        VC_BUILD=${VC_BUILD:-$VC_67_VERSION}
+        ;;
+esac
+
+LOG_UPLOAD_DEST="${LOG_UPLOAD_DEST:-${DEFAULT_LOG_UPLOAD_DEST}}"
+
+n=0 && rm -f "${input}"
+until [ $n -ge 5 -o -f "${input}" ]; do
+    echo "Retry.. $n"
+    echo "Downloading gcp file ${input}"
+    wget -nv https://storage.googleapis.com/vic-engine-builds/${input}
+
+    ((n++))
+    sleep 15
+done
+
+echo "Extracting .tar.gz"
+mkdir bin && tar xvzf ${input} -C bin/ --strip 1
+
+if [ ! -f  "bin/vic-machine-linux" ]; then
     echo "Tarball extraction failed..quitting the run"
-    break
+    rm -rf bin
+    exit
 else
+    VCH_COMMIT=$(bin/vic-machine-linux version | awk -F '-' '{print $NF}')
     echo "Tarball extraction passed, Running nightlies test.."
 fi
 
-if [[ $target == "6.0" ]]; then
-    echo "Executing nightly tests on vSphere 6.0"
-    pabot --processes 4 --removekeywords TAG:secret --exclude nsx --variable ESX_VERSION:$ESX_60_VERSION --variable VC_VERSION:$VC_60_VERSION -d 60/$i tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests
-    cat 60/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log
-elif [[ $target == "6.5" ]]; then
-    echo "Executing nightly tests on vSphere 6.5"
-    pabot --processes 4 --removekeywords TAG:secret -d 65/$i tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests
-    cat 65/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log
-elif [[ $target == "6.7" ]]; then
-    echo "Executing nightly tests on vSphere 6.7"
-    pabot --processes 4 --removekeywords TAG:secret --exclude nsx --exclude hetero --variable ESX_VERSION:$ESX_67_VERSION --variable VC_VERSION:$VC_67_VERSION -d 67/$i tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests
-    cat 67/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log
+
+pabot --processes 4 --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases}"
+cat ${target}/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log
+
+# See if any VMs leaked
+# TODO: should be a warning until clean, then changed to a failure if any leak
+echo "There should not be any VMs listed here"
+echo "======================================="
+timeout 60s sshpass -p ${NIMBUS_PASSWORD} ssh -o StrictHostKeyChecking\=no ${NIMBUS_USER}@${NIMBUS_GW} nimbus-ctl list
+echo "======================================="
+echo "If VMs are listed we should investigate why they are leaking"
+
+# archive the logs
+logarchive="logs_vch-${VCH_BUILD}-${VCH_COMMIT}_test-${BUILD_ID}-${GIT_COMMIT}_${BUILD_TIMESTAMP}.zip"
+/usr/bin/zip -9 -r "${logarchive}" "${target}" *.zip *.log *.debug *.tgz
+if [ $? -eq 0 ]; then
+    tests/nightly/upload-logs.sh ${logarchive} ${LOG_UPLOAD_DEST}
 fi
+
 # Pretty up the email results
 sed -i -e 's/^/<br>/g' console.log
 sed -i -e 's|PASS|<font color="green">PASS</font>|g' console.log
 sed -i -e 's|FAIL|<font color="red">FAIL</font>|g' console.log
 
-# See if any VMs leaked
-timeout 60s sshpass -p $NIMBUS_PASSWORD ssh -o StrictHostKeyChecking\=no $NIMBUS_USER@$NIMBUS_GW nimbus-ctl list
-
-tests/nightly/upload-logs.sh $target_$BUILD_TIMESTAMP
diff --git a/tests/nightly/upload-logs.sh b/tests/nightly/upload-logs.sh
index 8811c5b69c..82e72afdf3 100755
--- a/tests/nightly/upload-logs.sh
+++ b/tests/nightly/upload-logs.sh
@@ -14,28 +14,23 @@
 # limitations under the License.
 #
 
-echo "Upload logs"
+source=${1:?A source file must be specified}
+# The target destination in cloud storage. If the destination ends with a / it will be treated as a directory.
+# If not it _may_ be treated as a directory or a file depending on current remote objects. see gsutil cp doc.
+dest=${2:?A bucket must be specified, eg vic-ci-logs, or vic-ci-logs/user/branch/}
 
-set -x
-gsutil version -l
-set +x
-
-outfile="vic_nightly_logs_"$1".zip"
-echo $outfile
+if [ ! -r "${source}" ]; then
+  echo "Specified source file does not exist or cannot be read: ${source}"
+  exit 1
+fi
 
-if [ -d "60" ]; then
-    /usr/bin/zip -9 -r $outfile 60 *.zip *.log *.debug *.tgz
-elif [ -d "65" ]; then
-    /usr/bin/zip -9 -r $outfile 65 *.zip *.log *.debug *.tgz
-elif [ -d "67" ]; then
-    /usr/bin/zip -9 -r $outfile 67 *.zip *.log *.debug *.tgz
-else
-    echo "No output directories to upload!"
-    exit 1
+if [ ${dest:0:1} == "/" ]; then
+  echo "Destination must start with a bucket name and no leading /"
+  exit 1
 fi
 
 # GC credentials
-keyfile=~/vic-ci-logs.key
+keyfile=~/${dest%%/*}.key
 botofile=~/.boto
 if [ ! -f $keyfile ]; then
     echo -en $GS_PRIVATE_KEY > $keyfile
@@ -50,16 +45,22 @@ if [ ! -f $botofile ]; then
     echo "default_project_id = $GS_PROJECT_ID" >> $botofile
 fi
 
-if [ -f "$outfile" ]; then
-  gsutil cp $outfile gs://vic-ci-logs
-  echo "----------------------------------------------"
+echo "----------------------------------------------"
+echo "Uploading logs to ${dest}"
+echo "----------------------------------------------"
+if gsutil cp "${source}" "gs://${dest}"; then
+  url="https://console.cloud.google.com/m/cloudstorage/b/${dest}/o/${source}?authuser=1"
+  echo "$url" > log-download.url
   echo "Download test logs here:"
-  echo "https://console.cloud.google.com/m/cloudstorage/b/vic-ci-logs/o/$outfile?authuser=1"
-  echo "----------------------------------------------"
+  echo "$url"
 else
-  echo "No log output file to upload"
+  echo "Log upload faled. Dumping gsutil version logic"
+  set -x
+  gsutil version -l
+  set +x
 fi
+echo "----------------------------------------------"
+
+rm -f $keyfile
+
 
-if [ -f "$keyfile" ]; then
-  rm -f $keyfile
-fi
diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index 0ffe35ad08..25138317fd 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -45,12 +45,6 @@ Fetch POD
       ${pod}=  Fetch From Left  ${out}  :
       [return]  ${pod}
 
-Custom Testbed Keepalive
-    [Tags]  secret
-    [Arguments]  ${folder}
-    ${out}=  Run Secret SSHPASS command  %{NIMBUS_USER}  '%{NIMBUS_PASSWORD}'  touch ${folder}
-    [Return]  ${out}
-
 Deploy Nimbus ESXi Server
     [Arguments]  ${user}  ${password}  ${version}=${ESX_VERSION}  ${tls_disabled}=True
     ${name}=  Evaluate  'ESX-' + str(random.randint(1000,9999)) + str(time.clock())  modules=random,time
@@ -192,13 +186,22 @@ Deploy Nimbus vCenter Server Async
     ${out}=  Run Secret SSHPASS command  %{NIMBUS_USER}  '%{NIMBUS_PASSWORD}'  '${NIMBUS_LOCATION} nimbus-vcvadeploy --lease=0.25 --vcvaBuild ${version} ${name}'
     [Return]  ${out}
 
+# Deploys a nimbus testbed based on the specified testbed spec and options
+# user [required] - nimbus user
+# password [required] - password for nimbus user
+# spec [optional] - name of spec file in tests/resources/nimbus-testbeds
+# args [optiona;] - args to pass into testbeddeploy
 Deploy Nimbus Testbed
-    [Arguments]  ${user}  ${password}  ${testbed}
+    [Arguments]  ${user}  ${password}  ${spec}=  ${args}=
+
     Open Connection  %{NIMBUS_GW}
     Wait Until Keyword Succeeds  2 min  30 sec  Login  ${user}  ${password}
 
+    Run Keyword Unless  ${spec} == ${EMPTY}  Set Test Variable  ${specarg}  --testbedSpecRubyFile ./%{BUILD_TAG}/testbeds/${spec}
+    Run Keyword Unless  ${spec} == ${EMPTY}  Put File  tests/resources/nimbus-testbeds/${spec}  destination=./%{BUILD_TAG}/testbeds/
+
     :FOR  ${IDX}  IN RANGE  1  5
-    \   ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-testbeddeploy --lease 0.25 ${testbed}
+    \   ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-testbeddeploy --lease 0.25 ${specarg} ${args}
     \   Log  ${out}
     \   # Make sure the deploy actually worked
     \   ${status}=  Run Keyword And Return Status  Should Contain  ${out}  "deployment_result"=>"PASS"
@@ -215,26 +218,27 @@ Kill Nimbus Server
     Log  ${out}
     Close connection
 
-Cleanup Nimbus PXE folder
-    [Arguments]  ${user}  ${password}
+Cleanup Nimbus Folders
+    [Arguments]  ${deletePXE}=${false}
     Open Connection  %{NIMBUS_GW}
-    Wait Until Keyword Succeeds  2 min  30 sec  Login  ${user}  ${password}
-    ${out}=  Execute Command  ${NIMBUS_LOCATION} rm -rf public_html/pxe/*
+    Wait Until Keyword Succeeds  2 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
+    Run Keyword If  ${deletePXE}  Execute Command  ${NIMBUS_LOCATION} rm -rf public_html/pxe/*
+    Execute Command  ${NIMBUS_LOCATION} rm -rf %{BUILD_TAG}
     Close connection
 
+# Cleans up a list of VMs and deletes the pxe folder on nimbus gateway
 Nimbus Cleanup
     [Arguments]  ${vm_list}  ${collect_log}=True  ${dontDelete}=${false}
-    Run Keyword If  ${collect_log}  Run Keyword And Continue On Failure  Gather Logs From Test Server
-    Run Keyword And Ignore Error  Cleanup Nimbus PXE folder  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
-    Return From Keyword If  ${dontDelete}
     ${list}=  Catenate  @{vm_list}
-    Run Keyword And Ignore Error  Kill Nimbus Server  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  ${list}
+    Run Keyword   Nimbus Cleanup Single VM  ${list}  ${collect_log}  ${dontDelete}  ${true}
 
+# Cleans up a vm (or space separated string list of vms) but does not delete pxe folder on nimbus gateway
 Nimbus Cleanup Single VM
-    [Arguments]  ${vm}  ${collect_log}=True  ${dontDelete}=${false}
+    [Arguments]  ${vms}  ${collect_log}=True  ${dontDelete}=${false}  ${deletePXE}=${false}
     Run Keyword If  ${collect_log}  Run Keyword And Continue On Failure  Gather Logs From Test Server
+    Run Keyword And Ignore Error  Cleanup Nimbus Folders  ${deletePXE}
     Return From Keyword If  ${dontDelete}
-    Run Keyword And Ignore Error  Kill Nimbus Server  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  ${vm}
+    Run Keyword And Ignore Error  Kill Nimbus Server  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  ${vms}
 
 Gather Host IPs
     ${out}=  Run  govc ls host/cls
@@ -409,13 +413,13 @@ Get Vsphere Version
     \   Run Keyword And Return If  ${status}  Fetch From Right  ${line}  ${SPACE}
 
 Deploy Simple NFS Testbed
-    [Arguments]  ${user}  ${password}  ${additional-args}=
+    [Arguments]  ${user}  ${password}  ${spec}=  ${args}=
     ${name}=  Evaluate  'NFS-' + str(random.randint(1000,9999)) + str(time.clock())  modules=random,time
     Log To Console  \nDeploying Nimbus NFS testbed: ${name}
     Open Connection  %{NIMBUS_GW}
     Wait Until Keyword Succeeds  2 min  30 sec  Login  ${user}  ${password}
 
-    ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-testbeddeploy --testbedName nfs --runName ${name} ${additional-args}
+    ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-testbeddeploy  spec=${spec}  args=--testbedName nfs --runName ${name} ${args}
     Log  ${out}
     # Make sure the deploy actually worked
     Should Contain  ${out}  ${name}.nfs.0' is up. IP:
@@ -488,7 +492,7 @@ Create Simple VC Cluster With Static IP
     Set Suite Variable  ${NIMBUS_LOCATION}  NIMBUS_LOCATION=wdc
     Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
     Log To Console  Create a new simple vc cluster with static ip support...
-    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster-with-static --testbedSpecRubyFile /dbc/pa-dbc1111/mhagen/nimbus-testbeds/testbeds/vic-simple-cluster-with-static.rb --runName ${name}
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  spec=vic-simple-cluster-with-static.rb  args=--noSupportBundles --plugin testng --vcvaBuild ${VC_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-simple-cluster-with-static --runName ${name}
     Log  ${out}
 
     Open Connection  %{NIMBUS_GW}

From 366f354f55c1373cdcd8d4013f0f86326a0d7e9e Mon Sep 17 00:00:00 2001
From: George Hicken <hickeng@users.noreply.github.com>
Date: Fri, 15 Jun 2018 21:54:59 -0700
Subject: [PATCH 26/31] Preserve quoting in test runner script (#8060) (#8151)

This is a follow on from #8047 and corrects the variable
quoting when passing though one script to another.

This has been tested via a branch build with multiple
test suites specified.

(cherry picked from commit 88d047f5f5bf706d95160b06913cf1cc733d1193)
---
 tests/nightly/jenkins-nightly-run.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh
index ec29aabfd8..1cb028579f 100755
--- a/tests/nightly/jenkins-nightly-run.sh
+++ b/tests/nightly/jenkins-nightly-run.sh
@@ -24,7 +24,7 @@ VC_67_VERSION="ob-8217866"
 
 DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs"
 DEFAULT_VCH_BUILD="*"
-DEFAULT_TESTCASES="tests/manual-test-cases/Group5-Functional-Tests tests/manual-test-cases/Group13-vMotion tests/manual-test-cases/Group21-Registries tests/manual-test-cases/Group23-Future-Tests"
+DEFAULT_TESTCASES=("tests/manual-test-cases/Group5-Functional-Tests" "tests/manual-test-cases/Group13-vMotion" "tests/manual-test-cases/Group21-Registries" "tests/manual-test-cases/Group23-Future-Tests")
 
 
 if [[ $1 != "6.0" && $1 != "6.5" && $1 != "6.7" ]]; then
@@ -36,8 +36,8 @@ fi
 target="$1"
 echo "Target version: ${target}"
 shift
-# Take the remaining CLI arguments as a test case list
-testcases="${*:-$DEFAULT_TESTCASES}"
+# Take the remaining CLI arguments as a test case list - this is treated as an array to preserve quoting when passing to pabot
+testcases=("${@:-${DEFAULT_TESTCASES[@]}}")
 
 # TODO: the version downloaded by this logic is not coupled with the tests that will be run against it. This should be altered to pull a version that matches the commit SHA of the tests
 # we will be running or similar mechanism.
@@ -90,7 +90,7 @@ else
 fi
 
 
-pabot --processes 4 --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases}"
+pabot --processes 4 --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases[@]}"
 cat ${target}/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log
 
 # See if any VMs leaked

From 1e32fa27c39e3bc0a8ebab7b84608287361cb665 Mon Sep 17 00:00:00 2001
From: George Hicken <hickeng@users.noreply.github.com>
Date: Sat, 16 Jun 2018 14:09:43 -0700
Subject: [PATCH 27/31] Integration test script refinement and fixes (#8063)
 (#8151)

1. correct link construction for download of log bundles
2. correct custom test bed specified case in nimbus utils
3. clean generated log file name

The spec arg has been moved to the end to reduce the changes
necessary in calls to the Nimbus Deploy Testbed keyword that do
not specify a custom testbed file.

(cherry picked from commit 0f68789114dd9fd3448a2d439d9eb570a1b73aec)
---
 tests/nightly/jenkins-nightly-run.sh |  8 ++++++--
 tests/nightly/upload-logs.sh         | 11 +++++++++--
 tests/resources/Nimbus-Util.robot    |  8 ++++----
 3 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh
index 1cb028579f..e786662956 100755
--- a/tests/nightly/jenkins-nightly-run.sh
+++ b/tests/nightly/jenkins-nightly-run.sh
@@ -26,6 +26,7 @@ DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs"
 DEFAULT_VCH_BUILD="*"
 DEFAULT_TESTCASES=("tests/manual-test-cases/Group5-Functional-Tests" "tests/manual-test-cases/Group13-vMotion" "tests/manual-test-cases/Group21-Registries" "tests/manual-test-cases/Group23-Future-Tests")
 
+VIC_BINARY_PREFIX="vic_"
 
 if [[ $1 != "6.0" && $1 != "6.5" && $1 != "6.7" ]]; then
     echo "Please specify a target version. One of: 6.0, 6.5, 6.7"
@@ -42,8 +43,11 @@ testcases=("${@:-${DEFAULT_TESTCASES[@]}}")
 # TODO: the version downloaded by this logic is not coupled with the tests that will be run against it. This should be altered to pull a version that matches the commit SHA of the tests
 # we will be running or similar mechanism.
 VCH_BUILD=${VCH_BUILD:-${DEFAULT_VCH_BUILD}}
-input=$(gsutil ls -l gs://vic-engine-builds/vic_${VCH_BUILD} | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | cut -d '/' -f 4)
-VCH_BUILD=${input:4}
+input=$(gsutil ls -l gs://vic-engine-builds/${VIC_BINARY_PREFIX}${VCH_BUILD} | grep -v TOTAL | sort -k2 -r | head -n1 | xargs | cut -d ' ' -f 3 | cut -d '/' -f 4)
+
+# strip prefix and suffix from archive filename
+VCH_BUILD=${input#${VIC_BINARY_PREFIX}}
+VCH_BUILD=${VCH_BUILD%%.*}
 
 # Enforce short SHA
 GIT_COMMIT=${GIT_COMMIT:0:7}
diff --git a/tests/nightly/upload-logs.sh b/tests/nightly/upload-logs.sh
index 82e72afdf3..ba68e71370 100755
--- a/tests/nightly/upload-logs.sh
+++ b/tests/nightly/upload-logs.sh
@@ -29,8 +29,15 @@ if [ ${dest:0:1} == "/" ]; then
   exit 1
 fi
 
+# extract first path element as bucket name
+bucket=${dest%%/*}
+# drop bucket portion from path
+path=${dest#*/}
+# drop trailing / if any
+path=${path%/}
+
 # GC credentials
-keyfile=~/${dest%%/*}.key
+keyfile=~/${bucket}.key
 botofile=~/.boto
 if [ ! -f $keyfile ]; then
     echo -en $GS_PRIVATE_KEY > $keyfile
@@ -49,7 +56,7 @@ echo "----------------------------------------------"
 echo "Uploading logs to ${dest}"
 echo "----------------------------------------------"
 if gsutil cp "${source}" "gs://${dest}"; then
-  url="https://console.cloud.google.com/m/cloudstorage/b/${dest}/o/${source}?authuser=1"
+  url="https://console.cloud.google.com/m/cloudstorage/b/${bucket}/o/${path}/${source}?authuser=1"
   echo "$url" > log-download.url
   echo "Download test logs here:"
   echo "$url"
diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index 25138317fd..15d4bd78f7 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -189,16 +189,16 @@ Deploy Nimbus vCenter Server Async
 # Deploys a nimbus testbed based on the specified testbed spec and options
 # user [required] - nimbus user
 # password [required] - password for nimbus user
+# args [optional] - args to pass into testbeddeploy
 # spec [optional] - name of spec file in tests/resources/nimbus-testbeds
-# args [optiona;] - args to pass into testbeddeploy
 Deploy Nimbus Testbed
-    [Arguments]  ${user}  ${password}  ${spec}=  ${args}=
+    [Arguments]  ${user}  ${password}  ${args}=  ${spec}=${EMPTY}
 
     Open Connection  %{NIMBUS_GW}
     Wait Until Keyword Succeeds  2 min  30 sec  Login  ${user}  ${password}
 
-    Run Keyword Unless  ${spec} == ${EMPTY}  Set Test Variable  ${specarg}  --testbedSpecRubyFile ./%{BUILD_TAG}/testbeds/${spec}
-    Run Keyword Unless  ${spec} == ${EMPTY}  Put File  tests/resources/nimbus-testbeds/${spec}  destination=./%{BUILD_TAG}/testbeds/
+    ${specarg}=  Set Variable If  '${spec}' == '${EMPTY}'  ${EMPTY}  --testbedSpecRubyFile ./%{BUILD_TAG}/testbeds/${spec}    
+    Run Keyword Unless  '${spec}' == '${EMPTY}'  Put File  tests/resources/nimbus-testbeds/${spec}  destination=./%{BUILD_TAG}/testbeds/
 
     :FOR  ${IDX}  IN RANGE  1  5
     \   ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-testbeddeploy --lease 0.25 ${specarg} ${args}

From 3c6ad7bbadc67672c9cc1c35100caea675732208 Mon Sep 17 00:00:00 2001
From: George Hicken <hickeng@users.noreply.github.com>
Date: Mon, 18 Jun 2018 19:37:39 -0700
Subject: [PATCH 28/31] Additional fixes to integration tests (#8069) (#8151)

This builds on #8047 and #8063 to address some outstanding issues left by
that work. In the case of the hetrogeneous and NFS tests, it is test
updates necessitated by the prior Nimbus-Util.robot updates.

In the case of the VSAN tests it switches to using a testbed descriptor
based off the vic-simple-cluster spec instead of what appeared to be a
library spec, testng/testbeds/vsan.rb.

This disables the complex vSAN testsuite at this time as I could not infer
what the complex case was testing, and it is not in the test doc. It's
possible that it's testing multiple vSAN datastores and our ability to use
either but that's pure speculation.

(cherry picked from commit d533414843240339febba4cb5af0bfc5809bea75)
---
 .../13-1-vMotion-VCH-Appliance.robot          |   2 +-
 .../13-2-vMotion-Container.robot              |   2 +-
 .../Group21-Registries/21-1-Whitelist.robot   |   2 +-
 .../Group21-Registries/21-2-Artifactory.robot |   2 +-
 .../5-1-Distributed-Switch.robot              |   2 +-
 .../5-10-Multiple-Datacenter.robot            |   2 +-
 .../5-11-Multiple-Cluster.robot               |   2 +-
 .../5-12-Multiple-VLAN.robot                  |   2 +-
 .../5-13-Invalid-ESXi-Install.robot           |   2 +-
 .../5-14-Remove-Container-OOB.robot           |   2 +-
 .../5-15-NFS-Datastore.robot                  |   2 +-
 .../5-17-FC-Datastore.robot                   |   2 +-
 .../5-18-Datastore-Cluster-SDRS.robot         |   2 +-
 .../Group5-Functional-Tests/5-2-Cluster.robot |   2 +-
 .../5-21-Datastore-Path.robot                 |   2 +-
 .../5-22-NFS-Volume.robot                     |   2 +-
 .../5-24-Non-vSphere-Local-Cluster.robot      |   2 +-
 .../5-25-OPS-User-Grant.robot                 |   2 +-
 .../5-26-Static-IP-Address.robot              |   2 +-
 .../5-27-Selenium-Grid.robot                  |   2 +-
 .../5-28-VICAdmin-Isolated.robot              |   2 +-
 .../5-3-Enhanced-Linked-Mode.robot            |   2 +-
 .../5-4-High-Availability.robot               |   2 +-
 .../5-5-Heterogeneous-ESXi.robot              |   8 +-
 .../5-6-1-VSAN-Simple.robot                   |  17 +--
 ...plex.robot => 5-6-2-VSAN-Complex.disabled} |   8 +-
 .../Group5-Functional-Tests/5-8-DRS.robot     |   2 +-
 tests/resources/Nimbus-Util.robot             |  31 +++-
 tests/resources/nimbus-testbeds/vic-vsan.rb   | 138 ++++++++----------
 29 files changed, 124 insertions(+), 126 deletions(-)
 rename tests/manual-test-cases/Group5-Functional-Tests/{5-6-2-VSAN-Complex.robot => 5-6-2-VSAN-Complex.disabled} (90%)

diff --git a/tests/manual-test-cases/Group13-vMotion/13-1-vMotion-VCH-Appliance.robot b/tests/manual-test-cases/Group13-vMotion/13-1-vMotion-VCH-Appliance.robot
index bbea1af144..be87812278 100644
--- a/tests/manual-test-cases/Group13-vMotion/13-1-vMotion-VCH-Appliance.robot
+++ b/tests/manual-test-cases/Group13-vMotion/13-1-vMotion-VCH-Appliance.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 13-1 - vMotion VCH Appliance
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Create a VSAN Cluster  vic-vmotion-13-1
+Suite Setup  Nimbus Suite Setup  Create a VSAN Cluster  vic-vmotion-13-1
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 Test Teardown  Run Keyword If Test Failed  Gather All vSphere Logs
 
diff --git a/tests/manual-test-cases/Group13-vMotion/13-2-vMotion-Container.robot b/tests/manual-test-cases/Group13-vMotion/13-2-vMotion-Container.robot
index ab4b640dbf..7a8f4e103e 100644
--- a/tests/manual-test-cases/Group13-vMotion/13-2-vMotion-Container.robot
+++ b/tests/manual-test-cases/Group13-vMotion/13-2-vMotion-Container.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 13-2 - vMotion Container
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Create a VSAN Cluster  vic-vmotion-13-2
+Suite Setup  Nimbus Suite Setup  Create a VSAN Cluster  vic-vmotion-13-2
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 Test Teardown  Run Keyword If Test Failed  Gather All vSphere Logs
 
diff --git a/tests/manual-test-cases/Group21-Registries/21-1-Whitelist.robot b/tests/manual-test-cases/Group21-Registries/21-1-Whitelist.robot
index b7ce2bb732..4476f7498a 100644
--- a/tests/manual-test-cases/Group21-Registries/21-1-Whitelist.robot
+++ b/tests/manual-test-cases/Group21-Registries/21-1-Whitelist.robot
@@ -16,7 +16,7 @@
 Documentation  Test 21-01 - Whitelist
 Resource  ../../resources/Util.robot
 Resource  ../../resources/Harbor-Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Setup Harbor
+Suite Setup  Nimbus Suite Setup  Setup Harbor
 Suite Teardown  Nimbus Cleanup  ${list}  ${false}
 Test Teardown  Run Keyword If Test Failed  Cleanup VIC Appliance On Test Server
 
diff --git a/tests/manual-test-cases/Group21-Registries/21-2-Artifactory.robot b/tests/manual-test-cases/Group21-Registries/21-2-Artifactory.robot
index c70fc900c4..b0974f6ba2 100644
--- a/tests/manual-test-cases/Group21-Registries/21-2-Artifactory.robot
+++ b/tests/manual-test-cases/Group21-Registries/21-2-Artifactory.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 21-2 - Artifactory
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Artifactory Setup
+Suite Setup  Nimbus Suite Setup  Artifactory Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot
index ba06059830..664b5a2099 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-1 - Distributed Switch
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Distributed Switch Setup
+Suite Setup  Nimbus Suite Setup  Distributed Switch Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Variables ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-10-Multiple-Datacenter.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-10-Multiple-Datacenter.robot
index e31246b970..1e929d2d1b 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-10-Multiple-Datacenter.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-10-Multiple-Datacenter.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-10 - Multiple Datacenters
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Multiple Datacenter Setup
+Suite Setup  Nimbus Suite Setup  Multiple Datacenter Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
index 24a90676c4..0951f3970c 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-11-Multiple-Cluster.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-11 - Multiple Clusters
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Multiple Cluster Setup
+Suite Setup  Nimbus Suite Setup  Multiple Cluster Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-11-multiple-cluster*'  ${false}
 Test Teardown  Cleanup VIC Appliance On Test Server
 
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-12-Multiple-VLAN.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-12-Multiple-VLAN.robot
index 4f0cdac74a..6e15c602b9 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-12-Multiple-VLAN.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-12-Multiple-VLAN.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-12 - Multiple VLAN
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Multiple VLAN Setup
+Suite Setup  Nimbus Suite Setup  Multiple VLAN Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 Test Teardown  Cleanup VIC Appliance On Test Server
 
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-13-Invalid-ESXi-Install.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-13-Invalid-ESXi-Install.robot
index 3fcae6b647..976ab77cc6 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-13-Invalid-ESXi-Install.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-13-Invalid-ESXi-Install.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-13 - Invalid ESXi Install
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Invalid ESXi Install Setup
+Suite Setup  Nimbus Suite Setup  Invalid ESXi Install Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-14-Remove-Container-OOB.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-14-Remove-Container-OOB.robot
index 32afdfcf7b..18c41b84c7 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-14-Remove-Container-OOB.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-14-Remove-Container-OOB.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-14 - Remove Container OOB
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Remove Container OOB Setup
+Suite Setup  Nimbus Suite Setup  Remove Container OOB Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot
index 3008e5f590..3a1b68cec9 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-15-NFS-Datastore.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-15 - NFS Datastore
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  NFS Datastore Setup
+Suite Setup  Nimbus Suite Setup  NFS Datastore Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup Single VM  '*5-15-nfs-datastore*'
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-17-FC-Datastore.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-17-FC-Datastore.robot
index 319d19952a..8c8ae10963 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-17-FC-Datastore.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-17-FC-Datastore.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-17 - FC Datastore
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  FC Datastore Setup
+Suite Setup  Nimbus Suite Setup  FC Datastore Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-18-Datastore-Cluster-SDRS.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-18-Datastore-Cluster-SDRS.robot
index e5a864be4a..af6881d428 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-18-Datastore-Cluster-SDRS.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-18-Datastore-Cluster-SDRS.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-18 - Datastore Cluster SDRS
 Resource  ../../resources/Util.robot
-#Suite Setup  Wait Until Keyword Succeeds  10x  10m  SDRS Datastore Setup
+#Suite Setup  Nimbus Suite Setup  SDRS Datastore Setup
 #Test Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot
index a19fffa6a7..f276979570 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-2 - Cluster
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Cluster Setup
+Suite Setup  Nimbus Suite Setup  Cluster Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-21-Datastore-Path.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-21-Datastore-Path.robot
index d2b4da24a1..a623de003a 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-21-Datastore-Path.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-21-Datastore-Path.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation    Test 5-21 - Datastore-Path
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Setup Suite ESX
+Suite Setup  Nimbus Suite Setup  Setup Suite ESX
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Variables ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
index 0590553911..b076ab57e8 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-22-NFS-Volume.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-22 - NFS Volume
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Setup ESX And NFS Suite
+Suite Setup  Nimbus Suite Setup  Setup ESX And NFS Suite
 Suite Teardown  Run Keyword And Ignore Error  NFS Volume Cleanup
 Test Teardown  Gather NFS Logs
 
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-24-Non-vSphere-Local-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-24-Non-vSphere-Local-Cluster.robot
index 98e71e1bb1..f2efb6fcd3 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-24-Non-vSphere-Local-Cluster.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-24-Non-vSphere-Local-Cluster.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-24 - Non vSphere Local Cluster
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Non vSphere Local Cluster Install Setup
+Suite Setup  Nimbus Suite Setup  Non vSphere Local Cluster Install Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 Test Teardown  Cleanup VIC Appliance On Test Server
 
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
index a1abe00ea4..b72667305c 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-25-OPS-User-Grant.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-25 - OPS-User-Grant
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Ops User Create
+Suite Setup  Nimbus Suite Setup  Ops User Create
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot
index 937569d72f..56e73b6786 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-26-Static-IP-Address.robot
@@ -22,7 +22,7 @@ Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 *** Keywords ***
 Setup VC With Static IP
     ${name}=  Evaluate  'vic-5-26-' + str(random.randint(1000,9999))  modules=random
-    Wait Until Keyword Succeeds  10x  10m  Create Simple VC Cluster With Static IP  ${name}
+    Nimbus Suite Setup  Create Simple VC Cluster With Static IP  ${name}
     
 *** Test Cases ***
 Test
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot
index 573baef0e7..67933f6684 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-27-Selenium-Grid.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-27 - Selenium Grid
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Selenium Grid Test Setup
+Suite Setup  Nimbus Suite Setup  Selenium Grid Test Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.robot
index bb7a9ccef7..85ea8f63ee 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-28-VICAdmin-Isolated.robot
@@ -25,7 +25,7 @@ Deploy Testbed With Static IP
 
 Setup VC With No WAN
     ${name}=  Evaluate  'vic-5-28-' + str(random.randint(1000,9999))  modules=random
-    Wait Until Keyword Succeeds  10x  10m  Create Simple VC Cluster With Static IP  ${name}
+    Nimbus Suite Setup  Create Simple VC Cluster With Static IP  ${name}
     Set Test Environment Variables
     
     Log To Console  Create a vch with a public network on a no-wan portgroup.
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-3-Enhanced-Linked-Mode.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-3-Enhanced-Linked-Mode.robot
index dd69786dff..17e61a50cd 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-3-Enhanced-Linked-Mode.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-3-Enhanced-Linked-Mode.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-3 - Enhanced Linked Mode
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Enhanced Link Mode Setup
+Suite Setup  Nimbus Suite Setup  Enhanced Link Mode Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot
index 31339708ed..18e73e63af 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-4 - High Availability
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  High Availability Setup
+Suite Setup  Nimbus Suite Setup  High Availability Setup
 Suite Teardown  Nimbus Cleanup  ${list}
 Test Teardown  Run Keyword If Test Failed  Gather vSphere Logs
 
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-5-Heterogeneous-ESXi.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-5-Heterogeneous-ESXi.robot
index 54eb917fbf..f6a95741fa 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-5-Heterogeneous-ESXi.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-5-Heterogeneous-ESXi.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-5 - Heterogeneous ESXi
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Heterogenous ESXi Setup
+Suite Setup  Nimbus Suite Setup  Heterogenous ESXi Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 Force Tags  hetero
 
@@ -27,15 +27,15 @@ Heterogenous ESXi Setup
     ${pid-vc}=  Deploy Nimbus vCenter Server Async  ${vc}
     Set Suite Variable  @{list}  %{NIMBUS_USER}-${vc}
 
-    Run Keyword And Ignore Error  Cleanup Nimbus PXE folder  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
+    Run Keyword And Ignore Error  Cleanup Nimbus Folders  ${deletePXE}=%{true}
     ${esx1}  ${esx1-ip}=  Deploy Nimbus ESXi Server  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  3029944
     Append To List  ${list}  ${esx1}
 
-    Run Keyword And Ignore Error  Cleanup Nimbus PXE folder  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
+    Run Keyword And Ignore Error  Cleanup Nimbus Folders  ${deletePXE}=%{true}
     ${esx2}  ${esx2-ip}=  Deploy Nimbus ESXi Server  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  5572656
     Append To List  ${list}  ${esx2}
 
-    Run Keyword And Ignore Error  Cleanup Nimbus PXE folder  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
+    Run Keyword And Ignore Error  Cleanup Nimbus Folders  ${deletePXE}=%{true}
     ${esx3}  ${esx3-ip}=  Deploy Nimbus ESXi Server  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
     Append To List  ${list}  ${esx3}
 
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot
index abb4e6bd85..33aa4c81ff 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-6-1-VSAN-Simple.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-6-1 - VSAN-Simple
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  Simple VSAN Setup
+Suite Setup  Nimbus Suite Setup  Simple VSAN Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 
 *** Keywords ***
@@ -24,28 +24,25 @@ Simple VSAN Setup
     Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
     ${name}=  Evaluate  'vic-vsan-' + str(random.randint(1000,9999))  modules=random
     Set Suite Variable  ${user}  %{NIMBUS_USER}
-    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --plugin testng --vcfvtBuildPath /dbc/pa-dbc1111/mhagen/ --noSupportBundles --vcvaBuild ${VC_VERSION} --esxPxeDir ${ESX_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-vsan-simple-pxeBoot-vcva --runName ${name}
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  spec=vic-vsan.rb  args=--plugin testng --noSupportBundles --vcvaBuild ${VC_VERSION} --esxPxeDir ${ESX_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-vsan-simple-pxeBoot-vcva --runName ${name}
+
+    Log  ${out}
     Should Contain  ${out}  "deployment_result"=>"PASS"
+
     ${out}=  Split To Lines  ${out}
     :FOR  ${line}  IN  @{out}
-    \   ${status}=  Run Keyword And Return Status  Should Contain  ${line}  .vcva-${VC_VERSION}' is up. IP:
+    \   ${status}=  Run Keyword And Return Status  Should Contain  ${line}  .vc.0' is up. IP:
     \   ${ip}=  Run Keyword If  ${status}  Fetch From Right  ${line}  ${SPACE}
     \   Run Keyword If  ${status}  Set Suite Variable  ${vc-ip}  ${ip}
     \   Exit For Loop If  ${status}
 
-    Set Suite Variable  @{list}  ${user}-${name}.vcva-${VC_VERSION}  ${user}-${name}.esx.0  ${user}-${name}.esx.1  ${user}-${name}.esx.2  ${user}-${name}.esx.3  ${user}-${name}.nfs.0  ${user}-${name}.iscsi.0
+    Set Suite Variable  @{list}  ${user}-${name}.vcva-${VC_VERSION}  ${user}-${name}.esx.0  ${user}-${name}.esx.1  ${user}-${name}.esx.2  ${user}-${name}.esx.3  ${user}-${name}.nfs.0
     
     Log To Console  Set environment variables up for GOVC
     Set Environment Variable  GOVC_URL  ${vc-ip}
     Set Environment Variable  GOVC_USERNAME  Administrator@vsphere.local
     Set Environment Variable  GOVC_PASSWORD  Admin\!23
 
-    Add Host To Distributed Switch  /vcqaDC/host/cls
-
-    Log To Console  Enable DRS and VSAN on the cluster
-    ${out}=  Run  govc cluster.change -drs-enabled /vcqaDC/host/cls
-    Should Be Empty  ${out}
-
     Log To Console  Deploy VIC to the VC cluster
     Set Environment Variable  TEST_URL_ARRAY  ${vc-ip}
     Set Environment Variable  TEST_USERNAME  Administrator@vsphere.local
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.disabled
similarity index 90%
rename from tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot
rename to tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.disabled
index 5c0306b646..a4dc6cc87f 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-6-2-VSAN-Complex.disabled
@@ -1,4 +1,4 @@
-# Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+# Copyright 2016-2018 VMware, Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-6-2 - VSAN-Complex
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  VSAN Complex Setup
+Suite Setup  Nimbus Suite Setup  VSAN Complex Setup
 Suite Teardown  Run Keyword And Ignore Error  Nimbus Cleanup  ${list}
 Force Tags  vsan-complex
 
@@ -25,9 +25,11 @@ VSAN Complex Setup
     Run Keyword And Ignore Error  Nimbus Cleanup  ${list}  ${false}
     ${name}=  Evaluate  'vic-vsan-complex-' + str(random.randint(1000,9999))  modules=random
     Set Suite Variable  ${user}  %{NIMBUS_USER}
-    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  --plugin testng --vcfvtBuildPath /dbc/pa-dbc1111/mhagen/ --noSupportBundles --vcvaBuild ${VC_VERSION} --esxPxeDir ${ESX_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-vsan-complex-pxeBoot-vcva --runName ${name}
+    ${out}=  Deploy Nimbus Testbed  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}  spec=vic-vsan.rb  args=--plugin testng --noSupportBundles --vcvaBuild ${VC_VERSION} --esxPxeDir ${ESX_VERSION} --esxBuild ${ESX_VERSION} --testbedName vic-vsan-complex-pxeBoot-vcva --runName ${name}
+
     Log  ${out}
     Should Contain  ${out}  "deployment_result"=>"PASS"
+
     ${out}=  Split To Lines  ${out}
     :FOR  ${line}  IN  @{out}
     \   ${status}=  Run Keyword And Return Status  Should Contain  ${line}  .vcva-${VC_VERSION}' is up. IP:
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot
index a6690e6e5c..c1d7ad9c7d 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot
@@ -15,7 +15,7 @@
 *** Settings ***
 Documentation  Test 5-8 - DRS
 Resource  ../../resources/Util.robot
-Suite Setup  Wait Until Keyword Succeeds  10x  10m  DRS Setup
+Suite Setup  Nimbus Suite Setup  DRS Setup
 Suite Teardown  Nimbus Cleanup  ${list}
 
 *** Keywords ***
diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index 15d4bd78f7..9564ee29e9 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -198,17 +198,17 @@ Deploy Nimbus Testbed
     Wait Until Keyword Succeeds  2 min  30 sec  Login  ${user}  ${password}
 
     ${specarg}=  Set Variable If  '${spec}' == '${EMPTY}'  ${EMPTY}  --testbedSpecRubyFile ./%{BUILD_TAG}/testbeds/${spec}    
-    Run Keyword Unless  '${spec}' == '${EMPTY}'  Put File  tests/resources/nimbus-testbeds/${spec}  destination=./%{BUILD_TAG}/testbeds/
 
     :FOR  ${IDX}  IN RANGE  1  5
+    \   Run Keyword Unless  '${spec}' == '${EMPTY}'  Put File  tests/resources/nimbus-testbeds/${spec}  destination=./%{BUILD_TAG}/testbeds/
     \   ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-testbeddeploy --lease 0.25 ${specarg} ${args}
     \   Log  ${out}
     \   # Make sure the deploy actually worked
     \   ${status}=  Run Keyword And Return Status  Should Contain  ${out}  "deployment_result"=>"PASS"
     \   Return From Keyword If  ${status}  ${out}
-    \   Log To Console  Nimbus deployment ${IDX} failed, trying again in 5 minutes
-    \   Sleep  5 minutes
-    Fail  Deploy Nimbus Testbed Failed 5 times over the course of more than 25 minutes
+    \   Log To Console  Nimbus deployment ${IDX} failed, trying again in 1 minute
+    \   Sleep  1 minutes
+    Fail  Deploy Nimbus Testbed Failed 5 times over the course of more than 5 minutes
 
 Kill Nimbus Server
     [Arguments]  ${user}  ${password}  ${name}
@@ -416,13 +416,14 @@ Deploy Simple NFS Testbed
     [Arguments]  ${user}  ${password}  ${spec}=  ${args}=
     ${name}=  Evaluate  'NFS-' + str(random.randint(1000,9999)) + str(time.clock())  modules=random,time
     Log To Console  \nDeploying Nimbus NFS testbed: ${name}
-    Open Connection  %{NIMBUS_GW}
-    Wait Until Keyword Succeeds  2 min  30 sec  Login  ${user}  ${password}
 
-    ${out}=  Execute Command  ${NIMBUS_LOCATION} nimbus-testbeddeploy  spec=${spec}  args=--testbedName nfs --runName ${name} ${args}
+    ${out}=  Deploy Nimbus Testbed  user=${user}  password=${password}  spec=${spec}  args=--testbedName nfs --runName ${name} ${args}
     Log  ${out}
-    # Make sure the deploy actually worked
+
+    # Make sure the deploy actually worked and all components are up
     Should Contain  ${out}  ${name}.nfs.0' is up. IP:
+    Should Contain  ${out}  ${name}.nfs.1' is up. IP:
+    Should Contain  ${out}  ${name}.esx.0' is up. IP:
 
     Open Connection  %{NIMBUS_GW}
     Wait Until Keyword Succeeds  10 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
@@ -571,3 +572,17 @@ Get Name of First Local Storage For Host
     ${datastores}=  Run  govc host.info -host ${host} -json | jq -r '.HostSystems[].Config.FileSystemVolume.MountInfo[].Volume | select (.Type\=\="VMFS") | select (.Local\=\=true) | .Name'
     @{datastores}=  Split To Lines  ${datastores}
     [Return]  @{datastores}[0]
+
+# Simple wrapper to Wait Until Keyword Succeeds that allows callers to:
+# * use default retry count and delay
+# * specify specific retry counts and delays
+# * executor can globally override the above via the environment variables:
+#   NIMBUS_RETRY_ATTEMPTS
+#   NIMBUS_RETRY_DELAY
+Nimbus Suite Setup
+    [Arguments]  ${keyword}  ${attempts}=1x  ${delay}=1m  @{varargs}
+
+    ${useAttempts}=  Get Environment Variable  NIMBUS_RETRY_ATTEMPTS  ${attempts}x
+    ${useDelay}=     Get Environment Variable  NIMBUS_RETRY_DELAY     ${delay}
+
+    Wait Until Keyword Succeeds  ${useAttempts}  ${useDelay}  ${keyword}  @{varargs}
diff --git a/tests/resources/nimbus-testbeds/vic-vsan.rb b/tests/resources/nimbus-testbeds/vic-vsan.rb
index f934be0384..c764e70d99 100644
--- a/tests/resources/nimbus-testbeds/vic-vsan.rb
+++ b/tests/resources/nimbus-testbeds/vic-vsan.rb
@@ -1,4 +1,6 @@
+
 oneGB = 1 * 1000 * 1000 # in KB
+
 $testbed = Proc.new do |type, esxStyle, vcStyle, dbType, location|
   esxStyle ||= 'pxeBoot'
   vcStyle ||= 'vpxInstall'
@@ -13,62 +15,58 @@
     nameParts << dbType
   end
 
-  if type == 'complex'
-    numEsx = 8
-    numCPUs = 4
-    vmsPerHost = 1
-    addHosts = 'vsan-complex-vcqafvt'
-  elsif type == 'simple'
-    numEsx = 4
-    numCPUs = 4
-    vmsPerHost = 0
-    addHosts = 'vsan-simple-vcqafvt'
-  elsif type == 'complexCPU'
-    numEsx = 8
-    numCPUs = 2
-    vmsPerHost = 1
-    addHosts = 'vsan-complex-vcqafvt'
-  else
-    numEsx = 4
-    numCPUs = 2
-    vmsPerHost = 0
-    addHosts = 'vsan-simple-vcqafvt'
-  end
   testbed = {
-    'name' => nameParts.join('-'),
-    'esx' => (0...numEsx).map do |i|
+    "name" => nameParts.join('-'),
+    "version" => 3,
+    "esx" => (0..3).map do | idx |
       {
-        'name' => "esx.#{i}",
-        'style' => esxStyle,
-        'disks' => [ 30 * oneGB, 30 * oneGB, 30 * oneGB],
+        "name" => "esx.#{idx}",
+        "vc" => "vc.0",
+        "style" => "pxeInstall",
+        "desiredPassword" => "e2eFunctionalTest",
+        'numMem' => 13 * 1024,
+        'numCPUs' => 4,
+        "disks" => [ 30 * oneGB, 30 * oneGB, 30 * oneGB],
         'freeLocalLuns' => 1,
         'freeSharedLuns' => 2,
-        'numMem' => 13 * 1024,
-        'numCPUs' => numCPUs,
         'ssds' => [ 5*oneGB ],
         'nics' => 2,
-        'staf' => false,
-        'desiredPassword' => 'e2eFunctionalTest',
         'vmotionNics' => ['vmk0'],
-        'mountNfs' => ['nfs.0'],
+        'staf' => false,
+        "mountNfs" => ["nfs.0"],
+        "clusterName" => "cls",
       }
     end,
-    'vc' => {
-      'type' => vcStyle,
-      'additionalScript' => [], # XXX: Create users
-      'addHosts' => addHosts,
-      'linuxCertFile' => TestngLauncher.vcvaCertFile,
-      'dbType' => dbType,
-    },
+
+    "nfs" => [
+      {
+        "name" => "nfs.0",
+        "type" => "NFS41"
+      }
+    ],
+
+    "vcs" => [
+      {
+        "name" => "vc.0",
+        "type" => "vcva",
+        "dcName" => "dc1",
+        "clusters" => [{"name" => "cls", "vsan" => true, "enableDrs" => true, "enableHA" => true}],
+        "addHosts" => "allInSameCluster",
+      }
+    ],
+
     'vsan' => true,
-    'postBoot' => Proc.new do |runId, testbedSpec, vc, esxList, iscsiList|
-      ovf = NimbusUtils.get_absolute_ovf("CentOS/VM_OVF10.ovf")
-      if location && $nimbusEnv && $nimbusEnv['NIMBUS'] =~ /gamma/
-        ovf = 'http://10.116.111.50/testwareTestCentOS/VM_OVF10.ovf'
-      end
+    'vsanEnable' => true, # pulled from test-vpx/testbeds/vc-on-vsan.rb
 
-      cloudVc = vc
-      vim = VIM.connect cloudVc.rbvmomiConnectSpec
+    "postBoot" => Proc.new do |runId, testbedSpec, vmList, catApi, logDir|
+      esxList = vmList['esx']
+        esxList.each do |host|
+          host.ssh do |ssh|
+            ssh.exec!("esxcli network firewall set -e false")
+          end
+      end
+      vc = vmList['vc'][0]
+      vim = VIM.connect vc.rbvmomiConnectSpec
       datacenters = vim.serviceInstance.content.rootFolder.childEntity.grep(RbVmomi::VIM::Datacenter)
       raise "Couldn't find a Datacenter precreated"  if datacenters.length == 0
       datacenter = datacenters.first
@@ -83,7 +81,7 @@
           :configSpec => {
             :name => "test-ds"
           },
-	    }
+            }
       ).wait_for_completion
       Log.info "Vds DSwitch created"
 
@@ -120,39 +118,25 @@
       ).wait_for_completion
       Log.info "bridge DPG created"
 
-      VcQaTestbedCommon.postBoot(
-        vmsPerHost,
-        NimbusUtils.get_absolute_ovf("CentOS/VM_OVF10.ovf"),
-        runId, testbedSpec, vc, esxList, iscsiList)
-    end,
-    'nfs' => [
-      {
-        'name' => "nfs.0"
-      },
-    ],
+      Log.info "Add hosts to the DVS"
+      onecluster_pnic_spec = [ VIM::DistributedVirtualSwitchHostMemberPnicSpec({:pnicDevice => 'vmnic1'}) ]
+      dvs_config = VIM::DVSConfigSpec({
+        :configVersion => dvs.config.configVersion,
+        :host => cluster.host.map do |host|
+        {
+          :operation => :add,
+          :host => host,
+          :backing => VIM::DistributedVirtualSwitchHostMemberPnicBacking({
+            :pnicSpec => onecluster_pnic_spec
+          })
+        }
+        end
+      })
+      dvs.ReconfigureDvs_Task(:spec => dvs_config).wait_for_completion
+      Log.info "Hosts added to DVS successfully"
+    end
   }
 
-  if dbType == 'mssql'
-    testbed['genericVM'] ||= []
-    testbed['genericVM'] << {
-      'name' => 'vc-mssql',
-      'type' => 'mssql2k8',
-    }
-    testbed['vc']['dbHost'] = 'vc-mssql'
-  end
-
-  testbed = VcQaTestbedCommon.addSharedDisks testbed, [10, 10, 20, 20], sharedStorageStyle   # 2 x 20gb shared vmfs, 2 x 10gb free luns as defined by 'freeSharedLuns', DON'T CHANGE THE ORDERING UNLESS YOU KNOW WHAT YOU'RE DOING!
-
   testbed
 end
 
-[:pxeBoot, :fullInstall].each do |esxStyle|
-  [:vpxInstall, :vcva].each do |vcStyle|
-    [:embedded, :mssql, :oracle].each do |dbType|
-      ['complex', 'simple', 'complexCPU', 'simpleCPU'].each do |type|
-        testbedSpec = $testbed.call(type, esxStyle, vcStyle, dbType)
-        Nimbus::TestbedRegistry.registerTestbed testbedSpec
-      end
-    end
-  end
-end

From a9147f381f9770478082d6db1cc18335501e28ec Mon Sep 17 00:00:00 2001
From: George Hicken <hickeng@users.noreply.github.com>
Date: Thu, 21 Jun 2018 12:23:38 -0700
Subject: [PATCH 29/31] Updates to suite&dvs retry logic, increased
 predictability (#8074) (#8151)

This commit attempts to improve predictability by normalizing error
checking, retry approaches, pxe boot directory, and parameterizing
previously hard-coded build config. The latter is included because it
allows the caller to make reasonable estimates about how the build will
run rather than having to dig through test code.

Previously there were retries of different durations wrapped around the
outside of the Add Host To Distributed Switch keyword. If it may need
retrying then the operation should do so to avoid this cluster, and those
cases where a retry was not present.

Reduce sleep in inner retry loops to fail faster and adds a check for exit
code in addition to output parsing for faster fail. The inner loops should
be parameterized or removed completely in favour of the outer retry
logic unless there's significant improvement in success probability from
retrying at that level however this commit does not embark on that
work.

(cherry picked from commit 1387ab15f05282c466982415716e63a6f3c1591e)
---
 .../5-1-Distributed-Switch.robot              |  2 +-
 .../Group5-Functional-Tests/5-2-Cluster.robot |  6 ++--
 .../5-4-High-Availability.robot               |  2 +-
 .../Group5-Functional-Tests/5-8-DRS.robot     |  2 +-
 tests/nightly/jenkins-nightly-run.sh          |  8 +++--
 tests/resources/Nimbus-Util.robot             | 29 ++++++++++++-------
 tests/resources/nimbus-testbeds/vic-vsan.rb   |  2 +-
 7 files changed, 31 insertions(+), 20 deletions(-)

diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot
index 664b5a2099..bdf0a92b19 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-1-Distributed-Switch.robot
@@ -62,7 +62,7 @@ Distributed Switch Setup
     :FOR  ${IDX}  IN RANGE  ${esx_number}
     \   ${out}=  Run  govc host.add -hostname=@{esx_ips}[${IDX}] -username=root -dc=${datacenter} -password=${NIMBUS_ESX_PASSWORD} -noverify=true
     \   Should Contain  ${out}  OK
-    \   Wait Until Keyword Succeeds  5x  15 seconds  Add Host To Distributed Switch  @{esx_ips}[${IDX}]
+    \   Add Host To Distributed Switch  @{esx_ips}[${IDX}]
 
     Log To Console  Deploy VIC to the VC cluster
     Set Environment Variable  TEST_URL_ARRAY  ${vc_ip}
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot
index f276979570..55e2120690 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-2-Cluster.robot
@@ -69,9 +69,9 @@ Cluster Setup
 
     Create Three Distributed Port Groups  ha-datacenter
 
-    Wait Until Keyword Succeeds  5x  1 minute  Add Host To Distributed Switch  /ha-datacenter/host/cls/@{esx_ips}[0]
-    Wait Until Keyword Succeeds  5x  1 minute  Add Host To Distributed Switch  /ha-datacenter/host/cls/@{esx_ips}[1]
-    Wait Until Keyword Succeeds  5x  1 minute  Add Host To Distributed Switch  /ha-datacenter/host/cls/@{esx_ips}[2]
+    Add Host To Distributed Switch  /ha-datacenter/host/cls/@{esx_ips}[0]
+    Add Host To Distributed Switch  /ha-datacenter/host/cls/@{esx_ips}[1]
+    Add Host To Distributed Switch  /ha-datacenter/host/cls/@{esx_ips}[2]
 
     Log To Console  Enable DRS on the cluster
     ${out}=  Run  govc cluster.change -drs-enabled /ha-datacenter/host/cls
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot
index 18e73e63af..02c52771c5 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-4-High-Availability.robot
@@ -136,7 +136,7 @@ High Availability Setup
     \   Should Contain  ${out}  OK
 
     Log To Console  Add all the hosts to the distributed switch
-    Wait Until Keyword Succeeds  5x  5min  Add Host To Distributed Switch  /${datacenter}/host/cls
+    Add Host To Distributed Switch  /${datacenter}/host/cls
 
     Log To Console  Enable HA and DRS on the cluster
     ${out}=  Run  govc cluster.change -drs-enabled -ha-enabled /${datacenter}/host/cls
diff --git a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot
index c1d7ad9c7d..a303471813 100644
--- a/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot
+++ b/tests/manual-test-cases/Group5-Functional-Tests/5-8-DRS.robot
@@ -63,7 +63,7 @@ DRS Setup
     Should Contain  ${out}  OK
 
     Log To Console  Add all the hosts to the distributed switch
-    Wait Until Keyword Succeeds  5x  5min  Add Host To Distributed Switch  /ha-datacenter/host/cls
+    Add Host To Distributed Switch  /ha-datacenter/host/cls
 
     Log To Console  Deploy VIC to the VC cluster
     Set Environment Variable  TEST_URL_ARRAY  ${vc-ip}
diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh
index e786662956..b9917c3bcb 100755
--- a/tests/nightly/jenkins-nightly-run.sh
+++ b/tests/nightly/jenkins-nightly-run.sh
@@ -26,8 +26,13 @@ DEFAULT_LOG_UPLOAD_DEST="vic-ci-logs"
 DEFAULT_VCH_BUILD="*"
 DEFAULT_TESTCASES=("tests/manual-test-cases/Group5-Functional-Tests" "tests/manual-test-cases/Group13-vMotion" "tests/manual-test-cases/Group21-Registries" "tests/manual-test-cases/Group23-Future-Tests")
 
+DEFAULT_PARALLEL_JOBS=4
+
 VIC_BINARY_PREFIX="vic_"
 
+PARALLEL_JOBS=${PARLLEL_JOBS:-${DEFAULT_PARALLEL_JOBS}}
+
+
 if [[ $1 != "6.0" && $1 != "6.5" && $1 != "6.7" ]]; then
     echo "Please specify a target version. One of: 6.0, 6.5, 6.7"
     exit 1
@@ -93,8 +98,7 @@ else
     echo "Tarball extraction passed, Running nightlies test.."
 fi
 
-
-pabot --processes 4 --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases[@]}"
+pabot --processes ${PARALLEL_JOBS} --removekeywords TAG:secret ${excludes} --variable ESX_VERSION:${ESX_BUILD} --variable VC_VERSION:${VC_BUILD} -d ${target} "${testcases[@]}"
 cat ${target}/pabot_results/*/stdout.txt | grep '::' | grep -E 'PASS|FAIL' > console.log
 
 # See if any VMs leaked
diff --git a/tests/resources/Nimbus-Util.robot b/tests/resources/Nimbus-Util.robot
index 9564ee29e9..2c2de194de 100644
--- a/tests/resources/Nimbus-Util.robot
+++ b/tests/resources/Nimbus-Util.robot
@@ -59,8 +59,8 @@ Deploy Nimbus ESXi Server
     \   ${status}=  Run Keyword And Return Status  Should Contain  ${out}  To manage this VM use
     \   Exit For Loop If  ${status}
     \   Log To Console  ${out}
-    \   Log To Console  Nimbus deployment ${IDX} failed, trying again in 5 minutes
-    \   Sleep  5 minutes
+    \   Log To Console  Nimbus deployment ${IDX} failed, trying again in 1 minutes
+    \   Sleep  1 minutes
 
     # Now grab the IP address and return the name and ip for later use
     @{out}=  Split To Lines  ${out}
@@ -117,6 +117,7 @@ Deploy Multiple Nimbus ESXi Servers in Parallel
     \    ${result}=  Wait For Process  ${pid}
     \    Log  ${result.stdout}
     \    Log  ${result.stderr}
+    \    Should Be Equal As Integers  ${result.rc}  0
 
     &{ips}=  Create Dictionary
     :FOR  ${name}  IN  @{names}
@@ -146,8 +147,8 @@ Deploy Nimbus vCenter Server
     \   # Make sure the deploy actually worked
     \   ${status}=  Run Keyword And Return Status  Should Contain  ${out}  Overall Status: Succeeded
     \   Exit For Loop If  ${status}
-    \   Log To Console  Nimbus deployment ${IDX} failed, trying again in 5 minutes
-    \   Sleep  5 minutes
+    \   Log To Console  Nimbus deployment ${IDX} failed, trying again in 1 minute
+    \   Sleep  1 minutes
 
     # Now grab the IP address and return the name and ip for later use
     @{out}=  Split To Lines  ${out}
@@ -187,6 +188,8 @@ Deploy Nimbus vCenter Server Async
     [Return]  ${out}
 
 # Deploys a nimbus testbed based on the specified testbed spec and options
+# Calls Nimbus Cleanup first as a precaution. Impact on concurrent builds
+# is unknown.
 # user [required] - nimbus user
 # password [required] - password for nimbus user
 # args [optional] - args to pass into testbeddeploy
@@ -194,6 +197,8 @@ Deploy Nimbus vCenter Server Async
 Deploy Nimbus Testbed
     [Arguments]  ${user}  ${password}  ${args}=  ${spec}=${EMPTY}
 
+    Run Keyword And Ignore Error  Cleanup Nimbus Folders  deletePxe=${true}
+
     Open Connection  %{NIMBUS_GW}
     Wait Until Keyword Succeeds  2 min  30 sec  Login  ${user}  ${password}
 
@@ -222,7 +227,9 @@ Cleanup Nimbus Folders
     [Arguments]  ${deletePXE}=${false}
     Open Connection  %{NIMBUS_GW}
     Wait Until Keyword Succeeds  2 min  30 sec  Login  %{NIMBUS_USER}  %{NIMBUS_PASSWORD}
-    Run Keyword If  ${deletePXE}  Execute Command  ${NIMBUS_LOCATION} rm -rf public_html/pxe/*
+    # TODO: this may need pabot shared resource locking around it for multiple jobs. We're likely making use of the
+    # retry paths currently but it's not good practice.
+    Run Keyword If  ${deletePXE}  Execute Command  ${NIMBUS_LOCATION} rm -rf public_html/pxe/* public_html/pxeinstall/*
     Execute Command  ${NIMBUS_LOCATION} rm -rf %{BUILD_TAG}
     Close connection
 
@@ -368,7 +375,7 @@ Setup Network For Simple VC Cluster
     ${out}=  Run  govc dvs.portgroup.add -nports 12 -dc=${datacenter} -dvs=test-ds bridge
     Should Contain  ${out}  OK
 
-    Wait Until Keyword Succeeds  10x  3 minutes  Add Host To Distributed Switch  /${datacenter}/host/${cluster}  test-ds
+    Add Host To Distributed Switch  /${datacenter}/host/${cluster}  test-ds
 
     Log To Console  Enable DRS on the cluster
     ${out}=  Run  govc cluster.change -drs-enabled /${datacenter}/host/${cluster}
@@ -396,7 +403,7 @@ Create Three Distributed Port Groups
 Add Host To Distributed Switch
     [Arguments]  ${host}  ${dvs}=test-ds
     Log To Console  \nAdd host(s) to the distributed switch
-    ${out}=  Run  govc dvs.add -dvs=${dvs} -pnic=vmnic1 ${host}
+    ${out}=  Wait Until Keyword Succeeds  10x  30s  Run  govc dvs.add -dvs=${dvs} -pnic=vmnic1 ${host}
     Should Contain  ${out}  OK
 
 Disable TLS On ESX Host
@@ -580,9 +587,9 @@ Get Name of First Local Storage For Host
 #   NIMBUS_RETRY_ATTEMPTS
 #   NIMBUS_RETRY_DELAY
 Nimbus Suite Setup
-    [Arguments]  ${keyword}  ${attempts}=1x  ${delay}=1m  @{varargs}
+    [Arguments]  ${keyword}  @{varargs}
 
-    ${useAttempts}=  Get Environment Variable  NIMBUS_RETRY_ATTEMPTS  ${attempts}x
-    ${useDelay}=     Get Environment Variable  NIMBUS_RETRY_DELAY     ${delay}
+    ${useAttempts}=  Get Environment Variable  NIMBUS_RETRY_ATTEMPTS  1
+    ${useDelay}=     Get Environment Variable  NIMBUS_RETRY_DELAY     1m
 
-    Wait Until Keyword Succeeds  ${useAttempts}  ${useDelay}  ${keyword}  @{varargs}
+    Wait Until Keyword Succeeds  ${useAttempts}x  ${useDelay}  ${keyword}  @{varargs}
diff --git a/tests/resources/nimbus-testbeds/vic-vsan.rb b/tests/resources/nimbus-testbeds/vic-vsan.rb
index c764e70d99..b747e2f960 100644
--- a/tests/resources/nimbus-testbeds/vic-vsan.rb
+++ b/tests/resources/nimbus-testbeds/vic-vsan.rb
@@ -22,7 +22,7 @@
       {
         "name" => "esx.#{idx}",
         "vc" => "vc.0",
-        "style" => "pxeInstall",
+        "style" => esxStyle,
         "desiredPassword" => "e2eFunctionalTest",
         'numMem' => 13 * 1024,
         'numCPUs' => 4,

From 42449975df04f24bcbc020048bca3e48a755dc2f Mon Sep 17 00:00:00 2001
From: George Hicken <hickeng@users.noreply.github.com>
Date: Tue, 10 Jul 2018 10:40:07 -0700
Subject: [PATCH 30/31] Update vsphere build numbers for scenario tests (#8136)
 (#8151)

This updates the build numbers used for scenario test setup to the latest
update relases of the respective vSphere versions for 6.0, 6.5, and 6.7
Adds perma-links to the version/build mappings for future convenience.

(cherry picked from commit 3af7fa612889d5d7107280a6b85354d05ffdd090)
---
 tests/nightly/jenkins-nightly-run.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh
index b9917c3bcb..76798874bb 100755
--- a/tests/nightly/jenkins-nightly-run.sh
+++ b/tests/nightly/jenkins-nightly-run.sh
@@ -13,12 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ESX_60_VERSION="ob-5251623"
-VC_60_VERSION="ob-5112509"
 
-ESX_65_VERSION="ob-7867845"
-VC_65_VERSION="ob-7867539"
+# Permalink to ESX build numbers/versions: https://kb.vmware.com/kb/2143832
+# Permalink to vCenter build numbers/versions: https://kb.vmware.com/kb/2143838
 
+# 6.0u3e
+ESX_60_VERSION="ob-7967664"
+VC_60_VERSION="ob-7924803"
+
+# 6.5u2b
+ESX_65_VERSION="ob-8935087"
+VC_65_VERSION="ob-8815520" 
+
+# 6.7
 ESX_67_VERSION="ob-8169922"
 VC_67_VERSION="ob-8217866"
 

From 018ae80853f38b6f451d1b1f1ebc5c7e26c158be Mon Sep 17 00:00:00 2001
From: George Hicken <hickeng@users.noreply.github.com>
Date: Thu, 12 Jul 2018 09:27:50 -0700
Subject: [PATCH 31/31] Scenario tests run on 6.0u3 instead of 6.0u3e (#8139)
 (#8151)

The scenario tests depend on a testbed definition that references nimbus
test modules. Those do not have an implementation available for 6.0u3e.

This corrects a mistake in commit 3af7fa6

(cherry picked from commit e0dd53e0e72531b36976839a79093a3cf8126411)
---
 tests/nightly/jenkins-nightly-run.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/nightly/jenkins-nightly-run.sh b/tests/nightly/jenkins-nightly-run.sh
index 76798874bb..fa8fe351d1 100755
--- a/tests/nightly/jenkins-nightly-run.sh
+++ b/tests/nightly/jenkins-nightly-run.sh
@@ -17,9 +17,9 @@
 # Permalink to ESX build numbers/versions: https://kb.vmware.com/kb/2143832
 # Permalink to vCenter build numbers/versions: https://kb.vmware.com/kb/2143838
 
-# 6.0u3e
-ESX_60_VERSION="ob-7967664"
-VC_60_VERSION="ob-7924803"
+# 6.0u3
+ESX_60_VERSION="ob-5050593"
+VC_60_VERSION="ob-5112509" # the cloudvm build corresponding to the vpx build that is noted in the kb link above
 
 # 6.5u2b
 ESX_65_VERSION="ob-8935087"