diff --git a/cmd/vic-machine/inspect/inspect.go b/cmd/vic-machine/inspect/inspect.go index 0c33ef35e1..571ef7fcec 100644 --- a/cmd/vic-machine/inspect/inspect.go +++ b/cmd/vic-machine/inspect/inspect.go @@ -168,13 +168,13 @@ func (i *Inspect) upgradeStatusMessage(ctx context.Context, vch *vm.VirtualMachi return } - upgrading, _, err := vch.UpgradeInProgress(ctx, management.UpgradePrefix) + upgrading, err := vch.VCHUpdateStatus(ctx) if err != nil { - log.Errorf("Unable to determine if upgrade is in progress: %s", err) + log.Errorf("Unable to determine if upgrade/configure is in progress: %s", err) return } if upgrading { - log.Info("Upgrade in progress") + log.Info("Upgrade/configure in progress") return } diff --git a/cmd/vic-machine/list/list.go b/cmd/vic-machine/list/list.go index 27f6690837..d045572087 100644 --- a/cmd/vic-machine/list/list.go +++ b/cmd/vic-machine/list/list.go @@ -192,7 +192,7 @@ func (l *List) upgradeStatusMessage(ctx context.Context, vch *vm.VirtualMachine, return "Up to date" } - upgrading, _, err := vch.UpgradeInProgress(ctx, management.UpgradePrefix) + upgrading, err := vch.VCHUpdateStatus(ctx) if err != nil { return fmt.Sprintf("Unknown: %s", err) } diff --git a/cmd/vic-machine/upgrade/upgrade.go b/cmd/vic-machine/upgrade/upgrade.go index be2dc15877..d3db8bb993 100644 --- a/cmd/vic-machine/upgrade/upgrade.go +++ b/cmd/vic-machine/upgrade/upgrade.go @@ -64,6 +64,11 @@ func (u *Upgrade) Flags() []cli.Flag { Usage: "Roll back VCH version to before the previous upgrade", Destination: &u.Rollback, }, + cli.BoolFlag{ + Name: "resetInProgressFlag", + Usage: "Reset the UpdateInProgress flag. Warning: Do not reset this flag if another upgrade/configure process is running", + Destination: &u.ResetInProgressFlag, + }, } target := u.TargetFlags() @@ -148,6 +153,41 @@ func (u *Upgrade) Run(clic *cli.Context) (err error) { log.Infof("") log.Infof("VCH ID: %s", vch.Reference().String()) + if u.ResetInProgressFlag { + if err = vch.SetVCHUpdateStatus(ctx, false); err != nil { + log.Error("Failed to reset UpdateInProgress flag") + log.Error(err) + return errors.New("upgrade failed") + } + log.Infof("Reset UpdateInProgress flag successfully") + return nil + } + + upgrading, err := vch.VCHUpdateStatus(ctx) + if err != nil { + log.Error("Unable to determine if upgrade/configure is in progress") + log.Error(err) + return errors.New("upgrade failed") + } + if upgrading { + log.Error("Upgrade failed: another upgrade/configure operation is in progress") + log.Error("If no other upgrade/configure process is running, use --resetInProgressFlag to reset the VCH upgrade/configure status") + return errors.New("upgrade failed") + } + + if err = vch.SetVCHUpdateStatus(ctx, true); err != nil { + log.Error("Failed to set UpdateInProgress flag to true") + log.Error(err) + return errors.New("upgrade failed") + } + + defer func() { + if err = vch.SetVCHUpdateStatus(ctx, false); err != nil { + log.Error("Failed to reset UpdateInProgress") + log.Error(err) + } + }() + vchConfig, err := executor.FetchAndMigrateVCHConfig(vch) if err != nil { log.Error("Failed to get Virtual Container Host configuration") diff --git a/lib/install/data/data.go b/lib/install/data/data.go index 8cd1d47073..889e6c7886 100644 --- a/lib/install/data/data.go +++ b/lib/install/data/data.go @@ -76,8 +76,9 @@ type Data struct { Timeout time.Duration - Force bool - UseRP bool + Force bool + UseRP bool + ResetInProgressFlag bool AsymmetricRouting bool diff --git a/pkg/vsphere/vm/vm.go b/pkg/vsphere/vm/vm.go index 2b9546c9d7..707064f488 100644 --- a/pkg/vsphere/vm/vm.go +++ b/pkg/vsphere/vm/vm.go @@ -21,6 +21,7 @@ import ( "fmt" "net/url" "path" + "strconv" "strings" "sync/atomic" @@ -32,10 +33,13 @@ import ( "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" + "github.com/vmware/vic/pkg/vsphere/extraconfig/vmomi" "github.com/vmware/vic/pkg/vsphere/session" "github.com/vmware/vic/pkg/vsphere/tasks" ) +const UpdateStatus = "UpdateInProgress" + type InvalidState struct { r types.ManagedObjectReference } @@ -359,20 +363,6 @@ func IsUpgradeSnapshot(node *types.VirtualMachineSnapshotTree, upgradePrefix str return node != nil && strings.HasPrefix(node.Name, upgradePrefix) } -// UpgradeInProgress tells if an upgrade has already been started based on snapshot name beginning with upgradePrefix -func (vm *VirtualMachine) UpgradeInProgress(ctx context.Context, upgradePrefix string) (bool, string, error) { - node, err := vm.GetCurrentSnapshotTree(ctx) - if err != nil { - return false, "", fmt.Errorf("Failed to check upgrade snapshot status: %s", err) - } - - if IsUpgradeSnapshot(node, upgradePrefix) { - return true, node.Name, nil - } - - return false, "", nil -} - func (vm *VirtualMachine) registerVM(ctx context.Context, path, name string, vapp, pool, host *types.ManagedObjectReference, vmfolder *object.Folder) (*object.Task, error) { log.Debugf("Register VM %s", name) @@ -560,3 +550,41 @@ func (vm *VirtualMachine) DatastoreReference(ctx context.Context) ([]types.Manag } return mvm.Datastore, nil } + +// VCHUpdateStatus tells if an upgrade/configure has already been started based on the UpdateInProgress flag in ExtraConfig +// It returns the error if the vm operation does not succeed +func (vm *VirtualMachine) VCHUpdateStatus(ctx context.Context) (bool, error) { + info, err := vm.FetchExtraConfig(ctx) + if err != nil { + log.Errorf("Unable to get vm ExtraConfig: %s", err) + return false, err + } + + if v, ok := info[UpdateStatus]; ok { + status, err := strconv.ParseBool(v) + if err != nil { + // If error occurs, the bool return value does not matter for the caller. + return false, fmt.Errorf("failed to parse %s to bool: %s", v, err) + } + return status, nil + } + + // If UpdateStatus is not found, it might be the case that no upgrade/configure has been done to this VCH before + return false, nil +} + +// SetVCHUpdateStatus sets the VCH update status in ExtraConfig +func (vm *VirtualMachine) SetVCHUpdateStatus(ctx context.Context, status bool) error { + info := make(map[string]string) + info[UpdateStatus] = strconv.FormatBool(status) + + s := &types.VirtualMachineConfigSpec{ + ExtraConfig: vmomi.OptionValueFromMap(info), + } + + _, err := vm.WaitForResult(ctx, func(ctx context.Context) (tasks.Task, error) { + return vm.Reconfigure(ctx, *s) + }) + + return err +} diff --git a/pkg/vsphere/vm/vm_test.go b/pkg/vsphere/vm/vm_test.go index 37dd7c700d..f1dfcfe014 100644 --- a/pkg/vsphere/vm/vm_test.go +++ b/pkg/vsphere/vm/vm_test.go @@ -32,6 +32,7 @@ import ( "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" "github.com/vmware/vic/lib/guest" + "github.com/vmware/vic/pkg/vsphere/extraconfig/vmomi" "github.com/vmware/vic/pkg/vsphere/session" "github.com/vmware/vic/pkg/vsphere/simulator" "github.com/vmware/vic/pkg/vsphere/sys" @@ -603,3 +604,178 @@ func TestWaitForResult(t *testing.T) { assert.True(t, called == 2, "task should be retried once") assert.True(t, !vmm.IsInvalidState(ctx), "vm state should be fixed") } + +// SetUpdateStatus sets the VCH upgrade/configure status. +func SetUpdateStatus(ctx context.Context, updateStatus string, vm *VirtualMachine) error { + info := make(map[string]string) + info[UpdateStatus] = updateStatus + + s := &types.VirtualMachineConfigSpec{ + ExtraConfig: vmomi.OptionValueFromMap(info), + } + + _, err := vm.WaitForResult(ctx, func(ctx context.Context) (tasks.Task, error) { + return vm.Reconfigure(ctx, *s) + }) + if err != nil { + return err + } + + return nil +} + +// TestVCHUpdateStatus tests if VCHUpdateStatus() could obtain the correct VCH upgrade/configure status +func TestVCHUpdateStatus(t *testing.T) { + ctx := context.Background() + + // Nothing VC specific in this test, so we use the simpler ESX model + model := simulator.ESX() + defer model.Remove() + err := model.Create() + if err != nil { + t.Fatal(err) + } + + server := model.Service.NewServer() + defer server.Close() + client, err := govmomi.NewClient(ctx, server.URL, true) + if err != nil { + t.Fatal(err) + } + + // Any VM will do + finder := find.NewFinder(client.Client, false) + vmo, err := finder.VirtualMachine(ctx, "/ha-datacenter/vm/*_VM0") + if err != nil { + t.Fatal(err) + } + + config := &session.Config{ + Service: server.URL.String(), + Insecure: true, + Keepalive: time.Duration(5) * time.Minute, + DatacenterPath: "", + DatastorePath: "/ha-datacenter/datastore/*", + HostPath: "/ha-datacenter/host/*/*", + PoolPath: "/ha-datacenter/host/*/Resources", + } + + s, err := session.NewSession(config).Connect(ctx) + if err != nil { + t.Fatal(err) + } + s.Populate(ctx) + vmm := NewVirtualMachine(ctx, s, vmo.Reference()) + + updateStatus, err := vmm.VCHUpdateStatus(ctx) + if err != nil { + t.Fatalf("ERROR: %s", err) + } + assert.False(t, updateStatus, "updateStatus should be false if UpdateInProgress is not set in the VCH's ExtraConfig") + + // Set UpdateInProgress to false + SetUpdateStatus(ctx, "false", vmm) + + updateStatus, err = vmm.VCHUpdateStatus(ctx) + if err != nil { + t.Fatalf("ERROR: %s", err) + } + assert.False(t, updateStatus, "updateStatus should be false since UpdateInProgress is set to false") + + // Set UpdateInProgress to true + SetUpdateStatus(ctx, "true", vmm) + + updateStatus, err = vmm.VCHUpdateStatus(ctx) + if err != nil { + t.Fatalf("ERROR: %s", err) + } + assert.True(t, updateStatus, "updateStatus should be true since UpdateInProgress is set to true") + + // Set UpdateInProgress to NonBool + SetUpdateStatus(ctx, "NonBool", vmm) + + updateStatus, err = vmm.VCHUpdateStatus(ctx) + if assert.Error(t, err, "An error was expected") { + assert.Contains(t, err.Error(), "failed to parse", "Error msg should contain 'failed to parse' since UpdateInProgress is set to NonBool") + } +} + +// TestSetVCHUpdateStatus tests if SetVCHUpdateStatus() could set the VCH upgrade/configure status correctly +func TestSetVCHUpdateStatus(t *testing.T) { + ctx := context.Background() + + // Nothing VC specific in this test, so we use the simpler ESX model + model := simulator.ESX() + defer model.Remove() + err := model.Create() + if err != nil { + t.Fatal(err) + } + + server := model.Service.NewServer() + defer server.Close() + client, err := govmomi.NewClient(ctx, server.URL, true) + if err != nil { + t.Fatal(err) + } + + // Any VM will do + finder := find.NewFinder(client.Client, false) + vmo, err := finder.VirtualMachine(ctx, "/ha-datacenter/vm/*_VM0") + if err != nil { + t.Fatal(err) + } + + config := &session.Config{ + Service: server.URL.String(), + Insecure: true, + Keepalive: time.Duration(5) * time.Minute, + DatacenterPath: "", + DatastorePath: "/ha-datacenter/datastore/*", + HostPath: "/ha-datacenter/host/*/*", + PoolPath: "/ha-datacenter/host/*/Resources", + } + + s, err := session.NewSession(config).Connect(ctx) + if err != nil { + t.Fatal(err) + } + s.Populate(ctx) + vmm := NewVirtualMachine(ctx, s, vmo.Reference()) + + // Set UpdateInProgress to true and then check status + err = vmm.SetVCHUpdateStatus(ctx, true) + if err != nil { + t.Fatalf("ERROR: %s", err) + } + + info, err := vmm.FetchExtraConfig(ctx) + if err != nil { + t.Fatalf("ERROR: %s", err) + } + + v, ok := info[UpdateStatus] + if ok { + assert.Equal(t, "true", v, "UpdateInProgress should be true") + } else { + t.Fatal("ERROR: UpdateInProgress does not exist in ExtraConfig") + } + + // Set UpdateInProgress to false and then check status + err = vmm.SetVCHUpdateStatus(ctx, false) + if err != nil { + t.Fatalf("ERROR: %s", err) + } + + info, err = vmm.FetchExtraConfig(ctx) + if err != nil { + t.Fatalf("ERROR: %s", err) + } + + v, ok = info[UpdateStatus] + if ok { + assert.Equal(t, "false", v, "UpdateInProgress should be false") + } else { + t.Fatal("ERROR: UpdateInProgress does not exist in ExtraConfig") + } +} diff --git a/tests/resources/VCH-Util.robot b/tests/resources/VCH-Util.robot index 7f1487c473..eb2bc9625b 100644 --- a/tests/resources/VCH-Util.robot +++ b/tests/resources/VCH-Util.robot @@ -206,6 +206,18 @@ Run VIC Machine Inspect Command ${rc} ${output}= Run Secret VIC Machine Inspect Command %{VCH-NAME} Get Docker Params ${output} ${true} +Inspect VCH + [Arguments] ${expected} + ${rc} ${output}= Run And Return Rc And Output bin/vic-machine-linux inspect --name=%{VCH-NAME} --target=%{TEST_URL} --thumbprint=%{TEST_THUMBPRINT} --user=%{TEST_USERNAME} --password=%{TEST_PASSWORD} --compute-resource=%{TEST_RESOURCE} + Should Be Equal As Integers ${rc} 0 + Should Contain ${output} ${expected} + +Check UpdateInProgress + [Arguments] ${expected} + ${rc} ${output}= Run And Return Rc And Output govc vm.info -e %{VCH-NAME} | grep UpdateInProgress + Should Be Equal As Integers ${rc} 0 + Should Contain ${output} ${expected} + Gather Logs From Test Server [Tags] secret Run Keyword And Continue On Failure Run zip %{VCH-NAME}-certs -r %{VCH-NAME} diff --git a/tests/test-cases/Group11-Upgrade/11-04-Upgrade-UpdateInProgress.md b/tests/test-cases/Group11-Upgrade/11-04-Upgrade-UpdateInProgress.md new file mode 100644 index 0000000000..62e591e243 --- /dev/null +++ b/tests/test-cases/Group11-Upgrade/11-04-Upgrade-UpdateInProgress.md @@ -0,0 +1,25 @@ +Test 11-04 - Upgrade UpdateInProgress +======= + +#Purpose: +To verify that vic-machine inspect could detect the upgrade status of a VCH + +#Environment: +This test requires that a vSphere server is running and available + +#Test Steps: +1. Download vic_7315.tar.gz from bintray +2. Deploy VIC 7315 to vsphere server +3. Set UpdateInProgress to true using govc +4. Upgrade VCH +5. Run vic-machine upgrade --resetInProgressFlag to reset UpdateInProgress to false +6. Upgrade VCH +7. Run vic-machine inspect to check the upgrade status of the VCH (this should run in parallel with step 6) +8. After step 3 finishes, run step 4 again. + +#Expected Outcome: +* In step 4, output should contain "Upgrade failed: another upgrade/configure operation is in progress" +* In step 5, output should contain "Reset UpdateInProgress flag successfully" +* In step 6, output should contain "Completed successfully" +* In step 7, output should contain "Upgrade/configure in progress" +* In step 8, output should not contain "Upgrade/configure in progress" diff --git a/tests/test-cases/Group11-Upgrade/11-04-Upgrade-UpdateInProgress.robot b/tests/test-cases/Group11-Upgrade/11-04-Upgrade-UpdateInProgress.robot new file mode 100644 index 0000000000..367328a2bf --- /dev/null +++ b/tests/test-cases/Group11-Upgrade/11-04-Upgrade-UpdateInProgress.robot @@ -0,0 +1,36 @@ +# Copyright 2017 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +*** Settings *** +Documentation Test 11-04 - Upgrade-UpdateInProgress +Suite Setup Install VIC with version to Test Server 7315 +Suite Teardown Clean up VIC Appliance And Local Binary +Resource ../../resources/Util.robot + +*** Test Cases *** +Upgrade VCH with UpdateInProgress + Run govc vm.change -vm=%{VCH-NAME} -e=UpdateInProgress=true + Check UpdateInProgress true + ${rc} ${output}= Run And Return Rc And Output bin/vic-machine-linux upgrade --debug 1 --name=%{VCH-NAME} --target=%{TEST_URL} --user=%{TEST_USERNAME} --password=%{TEST_PASSWORD} --force=true --compute-resource=%{TEST_RESOURCE} --timeout %{TEST_TIMEOUT} + Should Contain ${output} Upgrade failed: another upgrade/configure operation is in progress + ${rc} ${output}= Run And Return Rc And Output bin/vic-machine-linux upgrade --resetInProgressFlag --name=%{VCH-NAME} --target=%{TEST_URL} --user=%{TEST_USERNAME} --password=%{TEST_PASSWORD} --force=true --compute-resource=%{TEST_RESOURCE} --timeout %{TEST_TIMEOUT} + Should Contain ${output} Reset UpdateInProgress flag successfully + Check UpdateInProgress false + +Upgrade and inspect VCH + Start Process bin/vic-machine-linux upgrade --debug 1 --name %{VCH-NAME} --target %{TEST_URL} --user %{TEST_USERNAME} --password %{TEST_PASSWORD} --force --compute-resource %{TEST_RESOURCE} --timeout %{TEST_TIMEOUT} shell=True alias=UpgradeVCH + Wait Until Keyword Succeeds 20x 5s Inspect VCH Upgrade/configure in progress + Wait For Process UpgradeVCH + Inspect VCH Completed successfully + Check UpdateInProgress false