Test test_vpn_client_package is failing randomly

[troydai]

======================================================================
ERROR: test_vpn_client_package (azure.cli.command_modules.network.tests.test_network_commands.NetworkVpnClientPackageScenarioTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/virtualenv/python2.7.13/lib/python2.7/site-packages/azure_devtools/scenario_tests/preparers.py", line 60, in _preparer_wrapper
    fn(test_class_instance, **kwargs)
  File "/home/travis/virtualenv/python2.7.13/lib/python2.7/site-packages/azure/cli/command_modules/network/tests/test_network_commands.py", line 1811, in test_vpn_client_package
    output = self.cmd('network vnet-gateway vpn-client generate -g {rg} -n {gateway}'.format(**kwargs)).get_output_in_json()
  File "/home/travis/virtualenv/python2.7.13/lib/python2.7/site-packages/azure/cli/testsdk/base.py", line 76, in cmd
    return execute(command, expect_failure=expect_failure).assert_with_checks(checks)
  File "/home/travis/virtualenv/python2.7.13/lib/python2.7/site-packages/azure/cli/testsdk/base.py", line 96, in __init__
    self._in_process_execute(command, expect_failure=expect_failure)
  File "/home/travis/virtualenv/python2.7.13/lib/python2.7/site-packages/azure/cli/testsdk/base.py", line 163, in _in_process_execute
    raise ex.exception
CLIError: Deployment failed. Tracking ID: 00000000-0000-0000-0000-000000000000. The subscription '00000000-0000-0000-0000-000000000000' could not be found.
-------------------- >> begin captured logging << --------------------
azure.cli.testsdk: INFO: Command "az group create --location westus --name cli_test_vpn_client_package000001 --tag use=az-test" => 0. Output: {
  "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001",
  "location": "westus",
  "managedBy": null,
  "name": "cli_test_vpn_client_package000001",
  "properties": {
    "provisioningState": "Succeeded"
  },
  "tags": {
    "use": "az-test"
  }
}
. 
azure.cli.testsdk: INFO: Command "network vnet create -g cli_test_vpn_client_package000001 -n vnet1 --subnet-name GatewaySubnet" => 0. Output: {
  "newVNet": {
    "addressSpace": {
      "addressPrefixes": [
        "10.0.0.0/16"
      ]
    },
    "dhcpOptions": {
      "dnsServers": []
    },
    "etag": "W/\"871604b4-ac33-48ce-8285-976b3383923d\"",
    "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/virtualNetworks/vnet1",
    "location": "westus",
    "name": "vnet1",
    "provisioningState": "Succeeded",
    "resourceGroup": "cli_test_vpn_client_package000001",
    "resourceGuid": "45a38e78-d771-4a4e-8072-c6b14f1217b4",
    "subnets": [
      {
        "addressPrefix": "10.0.0.0/24",
        "etag": "W/\"871604b4-ac33-48ce-8285-976b3383923d\"",
        "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/GatewaySubnet",
        "ipConfigurations": null,
        "name": "GatewaySubnet",
        "networkSecurityGroup": null,
        "provisioningState": "Succeeded",
        "resourceGroup": "cli_test_vpn_client_package000001",
        "resourceNavigationLinks": null,
        "routeTable": null,
        "serviceEndpoints": null
      }
    ],
    "tags": {},
    "type": "Microsoft.Network/virtualNetworks",
    "virtualNetworkPeerings": []
  }
}
. 
azure.cli.testsdk: INFO: Command "network public-ip create -g cli_test_vpn_client_package000001 -n pip1" => 0. Output: {
  "publicIp": {
    "dnsSettings": null,
    "etag": "W/\"00ff4702-8d4d-4a51-9e8b-a6e78ed4202f\"",
    "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/publicIPAddresses/pip1",
    "idleTimeoutInMinutes": 4,
    "ipAddress": null,
    "ipConfiguration": null,
    "location": "westus",
    "name": "pip1",
    "provisioningState": "Succeeded",
    "publicIpAddressVersion": "IPv4",
    "publicIpAllocationMethod": "Dynamic",
    "resourceGroup": "cli_test_vpn_client_package000001",
    "resourceGuid": "909bb1de-0cb5-4260-b2f0-1b0878986929",
    "sku": {
      "name": "Basic"
    },
    "tags": null,
    "type": "Microsoft.Network/publicIPAddresses"
  }
}
. 
azure.cli.testsdk: INFO: Command "network vnet-gateway create -g cli_test_vpn_client_package000001 -n vgw1 --address-prefix 100.1.1.0/24 --vnet vnet1 --public-ip-address pip1" => 0. Output: {
  "vnetGateway": {
    "activeActive": false,
    "bgpSettings": {
      "asn": 65515,
      "bgpPeeringAddress": "10.0.0.254",
      "peerWeight": 0
    },
    "enableBgp": false,
    "etag": "W/\"71e04f6d-19f7-405a-9f86-536d62ff52a5\"",
    "gatewayDefaultSite": null,
    "gatewayType": "Vpn",
    "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/virtualNetworkGateways/vgw1",
    "ipConfigurations": [
      {
        "etag": "W/\"71e04f6d-19f7-405a-9f86-536d62ff52a5\"",
        "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/virtualNetworkGateways/vgw1/ipConfigurations/vnetGatewayConfig0",
        "name": "vnetGatewayConfig0",
        "privateIpAllocationMethod": "Dynamic",
        "provisioningState": "Succeeded",
        "publicIpAddress": {
          "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/publicIPAddresses/pip1",
          "resourceGroup": "cli_test_vpn_client_package000001"
        },
        "resourceGroup": "cli_test_vpn_client_package000001",
        "subnet": {
          "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/GatewaySubnet",
          "resourceGroup": "cli_test_vpn_client_package000001"
        }
      }
    ],
    "location": "westus",
    "name": "vgw1",
    "provisioningState": "Succeeded",
    "resourceGroup": "cli_test_vpn_client_package000001",
    "resourceGuid": "1f73ae64-b167-4f74-9f0e-504f13b440fb",
    "sku": {
      "capacity": 2,
      "name": "Basic",
      "tier": "Basic"
    },
    "tags": null,
    "type": "Microsoft.Network/virtualNetworkGateways",
    "vpnClientConfiguration": {
      "radiusServerAddress": null,
      "radiusServerSecret": null,
      "vpnClientAddressPool": {
        "addressPrefixes": [
          "100.1.1.0/24"
        ]
      },
      "vpnClientProtocols": [],
      "vpnClientRevokedCertificates": [],
      "vpnClientRootCertificates": []
    },
    "vpnType": "RouteBased"
  }
}
. 
azure.cli.testsdk: INFO: Command "network vnet-gateway root-cert create -g cli_test_vpn_client_package000001 --gateway-name vgw1 -n cert1 --public-cert-data "/home/travis/virtualenv/python2.7.13/lib/python2.7/site-packages/azure/cli/command_modules/network/tests/test-root-cert.cer"" => 0. Output: {
  "activeActive": false,
  "bgpSettings": {
    "asn": 65515,
    "bgpPeeringAddress": "10.0.0.254",
    "peerWeight": 0
  },
  "enableBgp": false,
  "etag": "W/\"a30c2a07-1396-4b39-a778-35e01b366638\"",
  "gatewayDefaultSite": null,
  "gatewayType": "Vpn",
  "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/virtualNetworkGateways/vgw1",
  "ipConfigurations": [
    {
      "etag": "W/\"a30c2a07-1396-4b39-a778-35e01b366638\"",
      "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/virtualNetworkGateways/vgw1/ipConfigurations/vnetGatewayConfig0",
      "name": "vnetGatewayConfig0",
      "privateIpAllocationMethod": "Dynamic",
      "provisioningState": "Succeeded",
      "publicIpAddress": {
        "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/publicIPAddresses/pip1",
        "resourceGroup": "cli_test_vpn_client_package000001"
      },
      "resourceGroup": "cli_test_vpn_client_package000001",
      "subnet": {
        "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/GatewaySubnet",
        "resourceGroup": "cli_test_vpn_client_package000001"
      }
    }
  ],
  "location": "westus",
  "name": "vgw1",
  "provisioningState": "Succeeded",
  "resourceGroup": "cli_test_vpn_client_package000001",
  "resourceGuid": "1f73ae64-b167-4f74-9f0e-504f13b440fb",
  "sku": {
    "capacity": 2,
    "name": "Basic",
    "tier": "Basic"
  },
  "tags": null,
  "type": "Microsoft.Network/virtualNetworkGateways",
  "vpnClientConfiguration": {
    "radiusServerAddress": null,
    "radiusServerSecret": null,
    "vpnClientAddressPool": {
      "addressPrefixes": [
        "100.1.1.0/24"
      ]
    },
    "vpnClientProtocols": [],
    "vpnClientRevokedCertificates": [],
    "vpnClientRootCertificates": [
      {
        "etag": "W/\"a30c2a07-1396-4b39-a778-35e01b366638\"",
        "id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/cli_test_vpn_client_package000001/providers/Microsoft.Network/virtualNetworkGateways/vgw1/vpnClientRootCertificates/cert1",
        "name": "cert1",
        "provisioningState": "Succeeded",
        "publicCertData": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlDNXpDQ0FjK2dBd0lCQWdJUVFmSlRjQ25lanBKRGtVbnVoaGxIVWpBTkJna3Foa2lHOXcwQkFRc0ZBREFXDQpNUlF3RWdZRFZRUUREQXRRTWxOU2IyOTBRMlZ5ZERBZUZ3MHhOekEwTURNeU1USXdNRFphRncweE9EQTBNRE15DQpNVFF3TURaYU1CWXhGREFTQmdOVkJBTU1DMUF5VTFKdmIzUkRaWEowTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGDQpBQU9DQVE4QU1JSUJDZ0tDQVFFQXR0OXhUWjg1amlmOGc4MG00QnJqL3UxU3B6d2JsTXU5b1VBVnU4djh2akpPDQpObWVOektzYjFHNVFtQXVnOXdpZnIxaGJMdGJwWFRRaTR2NU9jOStEbGdyNDRlZTJQdkhLZGNVcnNFdGdBeXJQDQpuM3g3TTgvWVRyUW5mMHRSVXJwNyt4QmZvYmZPSjE2cHJZc09ieW56QlJNMXR1aG14OHZrajF2S2VlbThaZ0c0DQpLRHdycmVSWkdjS3huS1M2c0tsVjBGYTF0MDVhV3IrNjFFSTI1ZEhac1NpajIwa29lZ2FvM3FzKzlQVGtDNUwyDQo5ZHBPR0tQWTRTdlJqSGlqQnJLbFJjZTJwUWR4U1JIOHgzaVdWaW9XNnZtYmp6ekRPZHZNNmJxbTlPK1ltYi90DQpHdXI3bjl2Rnp2WjlMYTNXS0pBZzZoTXZQNjJOYXRUK1NUdWJZY1BrTXdJREFRQUJvekV3THpBT0JnTlZIUThCDQpBZjhFQkFNQ0FnUXdIUVlEVlIwT0JCWUVGSFpxeUpwZXlVSEQ0dDh2UFR5cVlSU0k2VkdpTUEwR0NTcUdTSWIzDQpEUUVCQ3dVQUE0SUJBUUJ5cW1TdXJDMVlHUHl0Yzl4SUVQVlg0WFZHVy9ZVVI1WmFKanhtMTE4M01FdEdZRFY4DQpKc29Ibmw4TE9QNzFsVFYwOHFtQlVBR2lzWEhnTWpyQ1FkUnFqc1hxeUNlZkFhbzY1czcyQTBwRFU3amtieWM5DQpIbzFycTNNbHJNZGEybGNHckFTVDUwVTlvbllQbEo1dUducTY5bWgwMjdnbWcvOUR1NXFRZFBNYUUvcGZWbmxlDQpNRFhUWjMrTjZxWWdkUmtERGl3WG9nZDNQWHVKdkc3NTlnR0NuZFhoUlRaaGlhbDRxNDdXK1Q2U2g4TC82WGxZDQo0cGM2ZERVZi9qUVExakdaSCtZNFliVDFXbnNWL2dRK2wwY0swUWpTT3k5aGo0K2ljeWZzZlRScWxwUmUyYXlwDQphcnYxYTBFanhHU1dWZlp2VGd4bkwyQWxOODJTbmEwQmNmdFANCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0NCg==",
        "resourceGroup": "cli_test_vpn_client_package000001"
      }
    ]
  },
  "vpnType": "RouteBased"
}
. 
--------------------- >> end captured logging << ---------------------

[troydai]

@tjprescott

[lmazuel]

Looks like what we thought of impact using mock no? ( @johanste )

[tjprescott]

Could be, though the impact we anticipated there was that something ELSE would fail if it called _poll while this test was running (and thus the mock in place). According to @yugangw-msft it only fails on Python 2.7. I'll check it out.

[troydai]

Is this issue addressed in any recent PR? It is still failing on master. And it will cause issues for the nightly build when Travis CI begin building them.

[yugangw-msft]

in case it helps, below is the output from the latest live run

/usr/local/lib/python3.6/site-packages/msrest/service_client.py:299: DeprecationWarning: Private attribute _client.add_header is deprecated. Use config.headers instead.
  DeprecationWarning)
/usr/local/lib/python3.6/site-packages/azure_devtools/scenario_tests/utilities.py:73: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()
  args, _, kw, _ = inspect.getargspec(fn)  # pylint: disable=deprecated-method
ERROR: Data for certificate /subscriptions/0b1f6471-1bf0-4dda-aec3-cb9272f09590/resourceGroups/cli_test_vpn_client_packageuj7zqjajhllgcrjykeznermd2bwyrre6kw6cewiaam6uqe6r/providers/Microsoft.Network/virtualNetworkGateways/vgw1/vpnClientRootCertificates/cert1 is invalid.
F
======================================================================
FAIL: test_vpn_client_package (azure.cli.command_modules.network.tests.latest.test_network_commands.NetworkVpnClientPackageScenarioTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/azure_devtools/scenario_tests/preparers.py", line 50, in _preparer_wrapper
    fn(test_class_instance, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/azure/cli/command_modules/network/tests/latest/test_network_commands.py", line 2020, in test_vpn_client_package
    self.cmd('network vnet-gateway root-cert create -g {rg} --gateway-name {gateway} -n {cert} --public-cert-data "{cert_path}"')
  File "/usr/local/lib/python3.6/site-packages/azure/cli/testsdk/base.py", line 176, in cmd
    return execute(self.cli_ctx, command, expect_failure=expect_failure).assert_with_checks(checks)
  File "/usr/local/lib/python3.6/site-packages/azure/cli/testsdk/base.py", line 203, in __init__
    raise AssertionError('The command failed. Exit code: {}'.format(self.exit_code))
AssertionError: The command failed. Exit code: 1

----------------------------------------------------------------------
Ran 1 test in 1523.979s

FAILED (failures=1)