From 00b9235e01441ebee812fc25d334381ef0132fe8 Mon Sep 17 00:00:00 2001 From: Daniel Miller Date: Mon, 27 Apr 2020 01:13:19 -0700 Subject: [PATCH 1/5] {Docs} Remove stale reference in README to closed issue about extensions (#12771) --- doc/extensions/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/extensions/README.md b/doc/extensions/README.md index 2b6d1a01aef..46dd325ddbf 100644 --- a/doc/extensions/README.md +++ b/doc/extensions/README.md @@ -12,8 +12,6 @@ What is an Extension? - Currently, we support one extension type, a [Python Wheel](http://pythonwheels.com/). - All extension documentation here refers to this type of extension. -> Extensions should be built with wheel `0.29.0` or `0.30.0` until [#6441](https://github.com/Azure/azure-cli/issues/6441) is resolved - What an Extension is not ------------------------ From 9e6c7e528af373aaca4d4bbe3de4c6e2f6c94d13 Mon Sep 17 00:00:00 2001 From: Zhenyu Zhou Date: Wed, 17 Jun 2020 19:23:00 +0800 Subject: [PATCH 2/5] Support hdinsight node reboot feature: Add two commands: az hdinsight host list and az hdinsight host restart --- .../hdinsight/_client_factory.py | 4 + .../cli/command_modules/hdinsight/_help.py | 23 + .../cli/command_modules/hdinsight/_params.py | 6 + .../cli/command_modules/hdinsight/commands.py | 12 + .../test_hdinsight_virtual_machine.yaml | 2152 +++++++++++++++++ .../tests/latest/test_hdinsight_commands.py | 85 +- src/azure-cli/requirements.py3.Darwin.txt | 2 +- src/azure-cli/requirements.py3.Linux.txt | 2 +- src/azure-cli/requirements.py3.windows.txt | 2 +- src/azure-cli/setup.py | 2 +- 10 files changed, 2256 insertions(+), 34 deletions(-) create mode 100644 src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/recordings/test_hdinsight_virtual_machine.yaml diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/_client_factory.py b/src/azure-cli/azure/cli/command_modules/hdinsight/_client_factory.py index d007d376287..588e57792e4 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/_client_factory.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/_client_factory.py @@ -77,3 +77,7 @@ def cf_hdinsight_extensions(cli_ctx, *_, **__): def cf_hdinsight_locations(cli_ctx, *_, **__): return cf_hdinsight(cli_ctx).locations + + +def cf_hdinsight_virtual_machines(cli_ctx, *_, **__): + return cf_hdinsight(cli_ctx).virtual_machines diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/_help.py b/src/azure-cli/azure/cli/command_modules/hdinsight/_help.py index 0cbdfc81c1e..97d33b47ef0 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/_help.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/_help.py @@ -162,6 +162,29 @@ --persist-on-success """ +helps['hdinsight host'] = """ +type: group +short-summary: Manage HDInsight cluster's virtual hosts. +""" + +helps['hdinsight host list'] = """ +type: command +short-summary: List the hosts of the specified HDInsight cluster. +examples: + - name: List the hosts of the specified HDInsight cluster. + text: |- + az hdinsight host list --resource-group MyResourceGroup --cluster-name MyCluster +""" + +helps['hdinsight host restart'] = """ +type: command +short-summary: Restart the specific hosts of the specified HDInsight cluster. +examples: + - name: Restart the specific hosts of the specified HDInsight cluster. + text: |- + az hdinsight host restart --resource-group MyResourceGroup --cluster-name MyCluster --hosts hostname +""" + helps['hdinsight wait'] = """ type: command short-summary: Place the CLI in a waiting state until an operation is complete. diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/_params.py b/src/azure-cli/azure/cli/command_modules/hdinsight/_params.py index 9f43fb8a683..68f646ee33d 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/_params.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/_params.py @@ -237,3 +237,9 @@ def load_arguments(self, _): c.argument('primary_key', help='The certificate for the Log Analytics workspace. ' 'Required when workspace ID is provided.') c.ignore('workspace_type') + + with self.argument_context('hdinsight host') as c: + c.argument('cluster_name', options_list=['--cluster-name'], + completer=get_resource_name_completion_list('Microsoft.HDInsight/clusters'), + help='The name of the cluster.') + c.argument('hosts', nargs="+", help='The host name of the cluster.') diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/commands.py b/src/azure-cli/azure/cli/command_modules/hdinsight/commands.py index 329612ca7ec..a676ec18f44 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/commands.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/commands.py @@ -12,6 +12,7 @@ def load_command_table(self, _): from ._client_factory import cf_hdinsight_locations from ._client_factory import cf_hdinsight_script_execution_history from ._client_factory import cf_hdinsight_script_actions + from ._client_factory import cf_hdinsight_virtual_machines hdinsight_clusters_sdk = CliCommandType( operations_tmpl='azure.mgmt.hdinsight.operations#ClustersOperations.{}', @@ -43,6 +44,11 @@ def load_command_table(self, _): client_factory=cf_hdinsight_script_execution_history ) + hdinsight_virtual_machines_sdk = CliCommandType( + operations_tmpl='azure.mgmt.hdinsight.operations#VirtualMachinesOperations.{}', + client_factory=cf_hdinsight_virtual_machines + ) + # cluster operations with self.command_group('hdinsight', hdinsight_clusters_sdk, client_factory=cf_hdinsight_clusters) as g: g.custom_command('create', 'create_cluster', supports_no_wait=True) @@ -92,3 +98,9 @@ def load_command_table(self, _): g.show_command('show', 'get_monitoring_status') g.custom_command('enable', 'enable_hdi_monitoring') g.command('disable', 'disable_monitoring') + + # VirtualMachine operations + with self.command_group('hdinsight host', hdinsight_virtual_machines_sdk, + client_factory=cf_hdinsight_virtual_machines) as g: + g.command('list', 'list_hosts') + g.command('restart', 'restart_hosts', confirmation=True, supports_no_wait=True) diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/recordings/test_hdinsight_virtual_machine.yaml b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/recordings/test_hdinsight_virtual_machine.yaml new file mode 100644 index 00000000000..ade9404e198 --- /dev/null +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/recordings/test_hdinsight_virtual_machine.yaml @@ -0,0 +1,2152 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-resource/10.0.0 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resources?$filter=resourceType%20eq%20%27Microsoft.Storage%2FstorageAccounts%27%20and%20name%20eq%20%27hdicli000002%27&api-version=2019-07-01 + response: + body: + string: '{"value":[],"nextLink":"https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resources?%24filter=resourceType+eq+%27Microsoft.Storage%2fstorageAccounts%27+and+name+eq+%27hdicli000002%27&api-version=2019-07-01&%24skiptoken=eyJuZXh0UGFydGl0aW9uS2V5IjoiMSE4IU5qZzFSVUUtIiwibmV4dFJvd0tleSI6IjEhMjI4IU9UWTBRekV3UWtJNFFUWkRORE5DUXpnelJETTJRak14T0VNMlF6Y3pNRFZmUjFKTUxVUkZSa0ZWVEZRNk1rUlRVVXc2TWtSWFJWTlVWVk10VFVsRFVrOVRUMFpVT2pKRlUxRk1PakpHVTBWU1ZrVlNVem95UmtRd09FNDNPVVJIVkVjNk1rWkVRVlJCUWtGVFJWTTZNa1pXTURNd01qSXhSak14TVRKRE0wVXpSVFF5TXpFNE1UaEJNRVU0T1VJeE9FTTNSa1EzU0VsV1JVMUZWRUZUVkU5U1JTMVhSVk5VVlZNLSJ9"}' + headers: + cache-control: + - no-cache + content-length: + - '643' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:27:07 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-resource/10.0.0 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resources?%24filter=resourceType+eq+%27Microsoft.Storage%2fstorageAccounts%27+and+name+eq+%27hdicli000002%27&api-version=2019-07-01&%24skiptoken=eyJuZXh0UGFydGl0aW9uS2V5IjoiMSE4IU5qZzFSVUUtIiwibmV4dFJvd0tleSI6IjEhMjI4IU9UWTBRekV3UWtJNFFUWkRORE5DUXpnelJETTJRak14T0VNMlF6Y3pNRFZmUjFKTUxVUkZSa0ZWVEZRNk1rUlRVVXc2TWtSWFJWTlVWVk10VFVsRFVrOVRUMFpVT2pKRlUxRk1PakpHVTBWU1ZrVlNVem95UmtRd09FNDNPVVJIVkVjNk1rWkVRVlJCUWtGVFJWTTZNa1pXTURNd01qSXhSak14TVRKRE0wVXpSVFF5TXpFNE1UaEJNRVU0T1VJeE9FTTNSa1EzU0VsV1JVMUZWRUZUVkU5U1JTMVhSVk5VVlZNLSJ9 + response: + body: + string: '{"value":[{"id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.Storage/storageAccounts/hdicli000002","name":"hdicli000002","type":"Microsoft.Storage/storageAccounts","sku":{"name":"Standard_LRS","tier":"Standard"},"kind":"Storage","location":"southcentralus","tags":{}}],"nextLink":"https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resources?%24filter=resourceType+eq+%27Microsoft.Storage%2fstorageAccounts%27+and+name+eq+%27hdicli000002%27&api-version=2019-07-01&%24skiptoken=eyJuZXh0UGFydGl0aW9uS2V5IjoiMSE4IU5qZzFSVUUtIiwibmV4dFJvd0tleSI6IjEhMTc2IU9UWTBRekV3UWtJNFFUWkRORE5DUXpnelJETTJRak14T0VNMlF6Y3pNRFZmUjFKTUxVaEZSMEZaUVUwNk1rUlRVRVU2TWtSVVJWTlVTVTVITFUxSlExSlBVMDlHVkRveVJVNUZWRmRQVWtzNk1rWk9SVlJYVDFKTFUwVkRWVkpKVkZsSFVrOVZVRk02TWtaVVJWTlVUVUZEU0VsT1JUb3lSRTVUUnkxRlFWTlVWVk0tIn0%3d"}' + headers: + cache-control: + - no-cache + content-length: + - '914' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:27:07 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-resource/10.0.0 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resources?%24filter=resourceType+eq+%27Microsoft.Storage%2fstorageAccounts%27+and+name+eq+%27hdicli000002%27&api-version=2019-07-01&%24skiptoken=eyJuZXh0UGFydGl0aW9uS2V5IjoiMSE4IU5qZzFSVUUtIiwibmV4dFJvd0tleSI6IjEhMTc2IU9UWTBRekV3UWtJNFFUWkRORE5DUXpnelJETTJRak14T0VNMlF6Y3pNRFZmUjFKTUxVaEZSMEZaUVUwNk1rUlRVRVU2TWtSVVJWTlVTVTVITFUxSlExSlBVMDlHVkRveVJVNUZWRmRQVWtzNk1rWk9SVlJYVDFKTFUwVkRWVkpKVkZsSFVrOVZVRk02TWtaVVJWTlVUVUZEU0VsT1JUb3lSRTVUUnkxRlFWTlVWVk0tIn0%3d + response: + body: + string: '{"value":[],"nextLink":"https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resources?%24filter=resourceType+eq+%27Microsoft.Storage%2fstorageAccounts%27+and+name+eq+%27hdicli000002%27&api-version=2019-07-01&%24skiptoken=eyJuZXh0UGFydGl0aW9uS2V5IjoiMSE4IU5qZzFSVUUtIiwibmV4dFJvd0tleSI6IjEhMTY0IU9UWTBRekV3UWtJNFFUWkRORE5DUXpnelJETTJRak14T0VNMlF6Y3pNRFZmUjFKTUxWTkJVMWRCVFZrNk1rUkVSVUpWUnkxTlNVTlNUMU5QUmxRNk1rVlRWRTlTUVVkRk9qSkdVMVJQVWtGSFJVRkRRMDlWVGxSVE9qSkdVMEZUVjBGTldVUkZRbFZIU0VSSlUxUlBVa0ZIUlMxRlFWTlVWVk0tIn0%3d"}' + headers: + cache-control: + - no-cache + content-length: + - '561' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:27:08 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-resource/10.0.0 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resources?%24filter=resourceType+eq+%27Microsoft.Storage%2fstorageAccounts%27+and+name+eq+%27hdicli000002%27&api-version=2019-07-01&%24skiptoken=eyJuZXh0UGFydGl0aW9uS2V5IjoiMSE4IU5qZzFSVUUtIiwibmV4dFJvd0tleSI6IjEhMTY0IU9UWTBRekV3UWtJNFFUWkRORE5DUXpnelJETTJRak14T0VNMlF6Y3pNRFZmUjFKTUxWTkJVMWRCVFZrNk1rUkVSVUpWUnkxTlNVTlNUMU5QUmxRNk1rVlRWRTlTUVVkRk9qSkdVMVJQVWtGSFJVRkRRMDlWVGxSVE9qSkdVMEZUVjBGTldVUkZRbFZIU0VSSlUxUlBVa0ZIUlMxRlFWTlVWVk0tIn0%3d + response: + body: + string: '{"value":[]}' + headers: + cache-control: + - no-cache + content-length: + - '12' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:27:08 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-storage/9.0.0 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.Storage/storageAccounts/hdicli000002?api-version=2019-06-01 + response: + body: + string: '{"sku":{"name":"Standard_LRS","tier":"Standard"},"kind":"Storage","id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.Storage/storageAccounts/hdicli000002","name":"hdicli000002","type":"Microsoft.Storage/storageAccounts","location":"southcentralus","tags":{},"properties":{"privateEndpointConnections":[],"networkAcls":{"bypass":"AzureServices","virtualNetworkRules":[],"ipRules":[],"defaultAction":"Allow"},"supportsHttpsTrafficOnly":true,"encryption":{"services":{"file":{"keyType":"Account","enabled":true,"lastEnabledTime":"2020-06-29T07:25:45.8212710Z"},"blob":{"keyType":"Account","enabled":true,"lastEnabledTime":"2020-06-29T07:25:45.8212710Z"}},"keySource":"Microsoft.Storage"},"provisioningState":"Succeeded","creationTime":"2020-06-29T07:25:45.7431395Z","primaryEndpoints":{"blob":"https://hdicli000002.blob.core.windows.net/","queue":"https://hdicli000002.queue.core.windows.net/","table":"https://hdicli000002.table.core.windows.net/","file":"https://hdicli000002.file.core.windows.net/"},"primaryLocation":"southcentralus","statusOfPrimary":"available"}}' + headers: + cache-control: + - no-cache + content-length: + - '1193' + content-type: + - application/json + date: + - Mon, 29 Jun 2020 07:27:08 GMT + expires: + - '-1' + pragma: + - no-cache + server: + - Microsoft-Azure-Storage-Resource-Provider/1.0,Microsoft-HTTPAPI/2.0 Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + Content-Length: + - '0' + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-storage/9.0.0 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.Storage/storageAccounts/hdicli000002/listKeys?api-version=2019-06-01 + response: + body: + string: '{"keys":[{"keyName":"key1","value":"veryFakedStorageAccountKey==","permissions":"FULL"},{"keyName":"key2","value":"veryFakedStorageAccountKey==","permissions":"FULL"}]}' + headers: + cache-control: + - no-cache + content-length: + - '288' + content-type: + - application/json + date: + - Mon, 29 Jun 2020 07:27:08 GMT + expires: + - '-1' + pragma: + - no-cache + server: + - Microsoft-Azure-Storage-Resource-Provider/1.0,Microsoft-HTTPAPI/2.0 Microsoft-HTTPAPI/2.0 + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-ratelimit-remaining-subscription-resource-requests: + - '11999' + status: + code: 200 + message: OK +- request: + body: 'b''{"location": "southcentralus", "properties": {"clusterVersion": "default", + "osType": "Linux", "clusterDefinition": {"kind": "spark", "configurations": + {"gateway": {"restAuthCredential.isEnabled": "true", "restAuthCredential.username": + "admin", "restAuthCredential.password": "Password1!"}}}, "computeProfile": {"roles": + [{"name": "headnode", "targetInstanceCount": 2, "hardwareProfile": {"vmSize": + "large"}, "osProfile": {"linuxOperatingSystemProfile": {"username": "sshuser", + "password": "Password1!"}}}, {"name": "workernode", "targetInstanceCount": 3, + "hardwareProfile": {"vmSize": "large"}, "osProfile": {"linuxOperatingSystemProfile": + {"username": "sshuser", "password": "Password1!"}}}]}, "storageProfile": {"storageaccounts": + [{"name": "hdicli000002.blob.core.windows.net", "isDefault": true, "container": + "default", "key": "veryFakedStorageAccountKey=="}]}}}''' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + Content-Length: + - '938' + Content-Type: + - application/json; charset=utf-8 + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: PUT + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003?api-version=2018-06-01-preview + response: + body: + string: '{"id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003","name":"hdicli-000003","type":"Microsoft.HDInsight/clusters","location":"South + Central US","etag":"34c735eb-e1d3-4805-909b-7e03518f2073","tags":null,"properties":{"clusterVersion":"4.0.1000.1","clusterHdpVersion":"","osType":"Linux","clusterDefinition":{"blueprint":"https://blueprints.azurehdinsight.net/spark-4.0.1000.1.2006192058.json","kind":"spark","componentVersion":{"spark":"2.4"}},"clusterId":"6b4b2988f6f447f9b88d7ba5b690eec4","computeProfile":{"roles":[{"name":"headnode","targetInstanceCount":2,"hardwareProfile":{"vmSize":"standard_a4_v2"},"osProfile":{"linuxOperatingSystemProfile":{"username":"sshuser"}},"encryptDataDisks":false},{"name":"workernode","targetInstanceCount":3,"hardwareProfile":{"vmSize":"standard_a4_v2"},"osProfile":{"linuxOperatingSystemProfile":{"username":"sshuser"}},"encryptDataDisks":false}]},"provisioningState":"InProgress","clusterState":"Accepted","createdDate":"2020-06-29T07:27:17.01","quotaInfo":{"coresUsed":20},"tier":"standard","encryptionInTransitProperties":{"isEncryptionInTransitEnabled":false},"storageProfile":{"storageaccounts":[{"name":"hdicli000002.blob.core.windows.net","resourceId":null,"msiResourceId":null,"key":null,"fileSystem":null,"container":"default","saskey":null,"isDefault":true,"fileshare":null}]},"minSupportedTlsVersion":"1.2","excludedServicesConfig":{"m_Item1":"default","m_Item2":""}}}' + headers: + azure-asyncoperation: + - https://management.azure.com:443/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + cache-control: + - no-cache + content-length: + - '1522' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:27:17 GMT + etag: + - '"34c735eb-e1d3-4805-909b-7e03518f2073"' + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-clusteruri: + - https://management.azure.com/subscriptions/964c10bb-8a6c-43bc-83d3-6b318c6c7305/resourceGroups/hdicli-oqs2m/providers/Microsoft.HDInsight/clusters/hdicli-z2znk7bj4?api-version=2018-06-01-preview + x-ms-hdi-served-by: + - southcentralus + x-ms-ratelimit-remaining-subscription-writes: + - '1199' + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:27:49 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:28:19 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:28:50 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:29:20 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:29:50 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:30:21 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:30:52 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:31:22 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:31:53 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:32:23 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:32:53 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:33:25 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:33:55 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:34:26 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:34:57 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:35:27 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:35:57 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:36:28 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:36:59 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:44:29 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:44:59 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:45:30 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:46:01 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:46:31 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:47:02 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:47:32 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:48:02 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:48:33 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:49:04 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:49:34 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:50:05 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"InProgress"}' + headers: + cache-control: + - no-cache + content-length: + - '23' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:50:35 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/create?api-version=2018-06-01-preview + response: + body: + string: '{"status":"Succeeded"}' + headers: + cache-control: + - no-cache + content-length: + - '22' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:51:06 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight create + Connection: + - keep-alive + ParameterSetName: + - -n -g -l -p -t --no-validation-timeout --storage-account --storage-container + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003?api-version=2018-06-01-preview + response: + body: + string: '{"id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003","name":"hdicli-000003","type":"Microsoft.HDInsight/clusters","location":"South + Central US","etag":"34c735eb-e1d3-4805-909b-7e03518f2073","tags":null,"properties":{"clusterVersion":"4.0.1000.1","clusterHdpVersion":"3.1.6.2-2","osType":"Linux","clusterDefinition":{"blueprint":"https://blueprints.azurehdinsight.net/spark-4.0.1000.1.2006192058.json","kind":"spark","componentVersion":{"spark":"2.4"}},"clusterId":"6b4b2988f6f447f9b88d7ba5b690eec4","computeProfile":{"roles":[{"name":"headnode","targetInstanceCount":2,"hardwareProfile":{"vmSize":"standard_a4_v2"},"osProfile":{"linuxOperatingSystemProfile":{"username":"sshuser"}},"encryptDataDisks":false},{"name":"workernode","targetInstanceCount":3,"hardwareProfile":{"vmSize":"standard_a4_v2"},"osProfile":{"linuxOperatingSystemProfile":{"username":"sshuser"}},"encryptDataDisks":false},{"name":"zookeepernode","targetInstanceCount":3,"hardwareProfile":{"vmSize":"standard_A2_v2"},"osProfile":{"linuxOperatingSystemProfile":{"username":"sshuser"}},"encryptDataDisks":false}]},"provisioningState":"Succeeded","clusterState":"Running","createdDate":"2020-06-29T07:27:17.01","quotaInfo":{"coresUsed":20},"connectivityEndpoints":[{"name":"SSH","protocol":"TCP","location":"hdicli-000003-ssh.azurehdinsight.net","port":22},{"name":"HTTPS","protocol":"TCP","location":"hdicli-000003.azurehdinsight.net","port":443}],"tier":"standard","encryptionInTransitProperties":{"isEncryptionInTransitEnabled":false},"storageProfile":{"storageaccounts":[{"name":"hdicli000002.blob.core.windows.net","resourceId":null,"msiResourceId":null,"key":null,"fileSystem":null,"container":"default","saskey":null,"isDefault":true,"fileshare":null}]},"minSupportedTlsVersion":"1.2","excludedServicesConfig":{"m_Item1":"default","m_Item2":""}}}' + headers: + cache-control: + - no-cache + content-length: + - '1931' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:51:07 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight show + Connection: + - keep-alive + ParameterSetName: + - -n -g + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003?api-version=2018-06-01-preview + response: + body: + string: '{"id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003","name":"hdicli-000003","type":"Microsoft.HDInsight/clusters","location":"South + Central US","etag":"34c735eb-e1d3-4805-909b-7e03518f2073","tags":null,"properties":{"clusterVersion":"4.0.1000.1","clusterHdpVersion":"3.1.6.2-2","osType":"Linux","clusterDefinition":{"blueprint":"https://blueprints.azurehdinsight.net/spark-4.0.1000.1.2006192058.json","kind":"spark","componentVersion":{"spark":"2.4"}},"clusterId":"6b4b2988f6f447f9b88d7ba5b690eec4","computeProfile":{"roles":[{"name":"headnode","targetInstanceCount":2,"hardwareProfile":{"vmSize":"standard_a4_v2"},"osProfile":{"linuxOperatingSystemProfile":{"username":"sshuser"}},"encryptDataDisks":false},{"name":"workernode","targetInstanceCount":3,"hardwareProfile":{"vmSize":"standard_a4_v2"},"osProfile":{"linuxOperatingSystemProfile":{"username":"sshuser"}},"encryptDataDisks":false},{"name":"zookeepernode","targetInstanceCount":3,"hardwareProfile":{"vmSize":"standard_A2_v2"},"osProfile":{"linuxOperatingSystemProfile":{"username":"sshuser"}},"encryptDataDisks":false}]},"provisioningState":"Succeeded","clusterState":"Running","createdDate":"2020-06-29T07:27:17.01","quotaInfo":{"coresUsed":20},"connectivityEndpoints":[{"name":"SSH","protocol":"TCP","location":"hdicli-000003-ssh.azurehdinsight.net","port":22},{"name":"HTTPS","protocol":"TCP","location":"hdicli-000003.azurehdinsight.net","port":443}],"tier":"standard","encryptionInTransitProperties":{"isEncryptionInTransitEnabled":false},"storageProfile":{"storageaccounts":[{"name":"hdicli000002.blob.core.windows.net","resourceId":null,"msiResourceId":null,"key":null,"fileSystem":null,"container":"default","saskey":null,"isDefault":true,"fileshare":null}]},"minSupportedTlsVersion":"1.2","excludedServicesConfig":{"m_Item1":"default","m_Item2":""}}}' + headers: + cache-control: + - no-cache + content-length: + - '1931' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:51:11 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight host list + Connection: + - keep-alive + Content-Length: + - '0' + ParameterSetName: + - --resource-group --cluster-name + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/listHosts?api-version=2018-06-01-preview + response: + body: + string: '[{"name":"gw0-hdicli"},{"name":"gw1-hdicli"},{"name":"hn0-hdicli"},{"name":"hn1-hdicli"},{"name":"wn0-hdicli"},{"name":"wn1-hdicli"},{"name":"wn4-hdicli"},{"name":"zk0-hdicli"},{"name":"zk5-hdicli"},{"name":"zk6-hdicli"}]' + headers: + cache-control: + - no-cache + content-length: + - '221' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:56:40 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-ratelimit-remaining-subscription-writes: + - '1199' + status: + code: 200 + message: OK +- request: + body: '["wn0-hdicli"]' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight host restart + Connection: + - keep-alive + Content-Length: + - '14' + Content-Type: + - application/json; charset=utf-8 + ParameterSetName: + - --resource-group --cluster-name --hosts --yes + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + accept-language: + - en-US + method: POST + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/restartHosts?api-version=2018-06-01-preview + response: + body: + string: '' + headers: + azure-asyncoperation: + - https://management.azure.com:443/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/1fe64ec8-87c7-43c5-bfd2-0ebdda307d29-0-r?api-version=2018-06-01-preview + cache-control: + - no-cache + content-length: + - '0' + date: + - Mon, 29 Jun 2020 07:56:55 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + x-content-type-options: + - nosniff + x-ms-ratelimit-remaining-subscription-writes: + - '1199' + status: + code: 202 + message: Accepted +- request: + body: null + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + CommandName: + - hdinsight host restart + Connection: + - keep-alive + ParameterSetName: + - --resource-group --cluster-name --hosts --yes + User-Agent: + - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 + azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 + method: GET + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/hdicli-000001/providers/Microsoft.HDInsight/clusters/hdicli-000003/azureasyncoperations/1fe64ec8-87c7-43c5-bfd2-0ebdda307d29-0-r?api-version=2018-06-01-preview + response: + body: + string: '{"status":"Succeeded"}' + headers: + cache-control: + - no-cache + content-length: + - '22' + content-type: + - application/json; charset=utf-8 + date: + - Mon, 29 Jun 2020 07:57:26 GMT + expires: + - '-1' + pragma: + - no-cache + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-content-type-options: + - nosniff + x-ms-hdi-served-by: + - southcentralus + status: + code: 200 + message: OK +version: 1 diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py index 6361ba7a57f..a53b2efd53a 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py @@ -12,7 +12,7 @@ class HDInsightClusterTests(ScenarioTest): - location = 'eastus2' + location = 'southcentralus' # Uses 'rg' kwarg @ResourceGroupPreparer(name_prefix='hdicli-', location=location, random_name_length=12) @@ -248,14 +248,15 @@ def test_hdinsight_script_action(self, storage_account_info): ]) # list script action history and validate script appears there. - script_actions = self.cmd('az hdinsight script-action list-execution-history -g {rg} --cluster-name {cluster}', checks=[ - self.check('type(@)', 'array'), - self.check('length(@)', 1), - self.check('[0].name', '{script_action}'), - self.check('[0].uri', '{script_uri}'), - self.check('[0].roles', roles), - self.check('[0].status', 'Succeeded') - ]).get_output_in_json() + script_actions = self.cmd('az hdinsight script-action list-execution-history -g {rg} --cluster-name {cluster}', + checks=[ + self.check('type(@)', 'array'), + self.check('length(@)', 1), + self.check('[0].name', '{script_action}'), + self.check('[0].uri', '{script_uri}'), + self.check('[0].roles', roles), + self.check('[0].status', 'Succeeded') + ]).get_output_in_json() # get the script action by ID and validate it's the same action. self.kwargs['script_execution_id'] = str(script_actions[0]['scriptExecutionId']) @@ -270,13 +271,14 @@ def test_hdinsight_script_action(self, storage_account_info): '--name {script_action_1} --script-uri {script_uri} --roles {head_node} {worker_node}') # list script action history and validate the new script also appears. - script_actions = self.cmd('az hdinsight script-action list-execution-history -g {rg} --cluster-name {cluster}', checks=[ - self.check('type(@)', 'array'), - self.check('length(@)', 2), - self.check('[0].name', '{script_action_1}'), - self.check("[0].uri", '{script_uri}'), - self.check("[0].status", 'Succeeded') - ]).get_output_in_json() + script_actions = self.cmd('az hdinsight script-action list-execution-history -g {rg} --cluster-name {cluster}', + checks=[ + self.check('type(@)', 'array'), + self.check('length(@)', 2), + self.check('[0].name', '{script_action_1}'), + self.check("[0].uri", '{script_uri}'), + self.check("[0].status", 'Succeeded') + ]).get_output_in_json() # promote non-persisted script. self.kwargs['script_execution_id'] = str(script_actions[0]['scriptExecutionId']) @@ -294,18 +296,41 @@ def test_hdinsight_script_action(self, storage_account_info): ]) # list script action history and validate both scripts are there. - script_actions = self.cmd('az hdinsight script-action list-execution-history -g {rg} --cluster-name {cluster}', checks=[ + script_actions = self.cmd('az hdinsight script-action list-execution-history -g {rg} --cluster-name {cluster}', + checks=[ + self.check('type(@)', 'array'), + self.check('length(@)', 2), + self.check('[0].name', '{script_action_1}'), + self.check("[0].uri", '{script_uri}'), + self.check("[0].roles", roles), + self.check("[0].status", 'Succeeded'), + self.check('[1].name', '{script_action}'), + self.check("[1].uri", '{script_uri}'), + self.check("[1].roles", roles), + self.check("[1].status", 'Succeeded') + ]) + + @ResourceGroupPreparer(name_prefix='hdicli-', location=location, random_name_length=12) + @StorageAccountPreparer(name_prefix='hdicli', location=location, parameter_name='storage_account') + def test_hdinsight_virtual_machine(self, storage_account_info): + self._create_hdinsight_cluster( + HDInsightClusterTests._wasb_arguments(storage_account_info) + ) + + # list hosts of the cluster + host_list = self.cmd('az hdinsight host list --resource-group {rg} --cluster-name {cluster}', checks=[ self.check('type(@)', 'array'), - self.check('length(@)', 2), - self.check('[0].name', '{script_action_1}'), - self.check("[0].uri", '{script_uri}'), - self.check("[0].roles", roles), - self.check("[0].status", 'Succeeded'), - self.check('[1].name', '{script_action}'), - self.check("[1].uri", '{script_uri}'), - self.check("[1].roles", roles), - self.check("[1].status", 'Succeeded') - ]) + self.exists('[0].name') + ]).get_output_in_json() + + target_host = host_list[0]['name'] + for host in host_list: + if host['name'].startswith('wn'): + target_host = host['name'] + break + self.kwargs['target_host'] = target_host + # restart host of the cluster + self.cmd('az hdinsight host restart --resource-group {rg} --cluster-name {cluster} --hosts {target_host} --yes') def _create_hdinsight_cluster(self, *additional_create_arguments): self.kwargs.update({ @@ -344,7 +369,7 @@ def _wasb_arguments(storage_account_info, specify_key=False, specify_container=T key_args = ' --storage-account-key "{}"'.format(storage_account_key) if specify_key else "" container_args = ' --storage-container {}'.format('default') if specify_container else "" - return '--storage-account {}{}{}'\ + return '--storage-account {}{}{}' \ .format(storage_account_name, key_args, container_args) @staticmethod @@ -354,13 +379,13 @@ def _kafka_arguments(): @staticmethod def _rest_proxy_arguments(): return '--kafka-management-node-size {} --kafka-client-group-id {} --kafka-client-group-name {} -v 4.0 ' \ - '--component-version {} --location {}'\ + '--component-version {} --location {}' \ .format('Standard_D4_v2', '7bef90fa-0aa3-4bb4-b4d2-2ae7c14cfe41', 'KafakaRestProperties', 'kafka=2.1', '"South Central US"') @staticmethod def _optional_data_disk_arguments(): - return '--workernode-data-disk-storage-account-type {} --workernode-data-disk-size {}'\ + return '--workernode-data-disk-storage-account-type {} --workernode-data-disk-size {}' \ .format('Standard_LRS', '1023') @staticmethod diff --git a/src/azure-cli/requirements.py3.Darwin.txt b/src/azure-cli/requirements.py3.Darwin.txt index d73769dc1f5..8948ca6b405 100644 --- a/src/azure-cli/requirements.py3.Darwin.txt +++ b/src/azure-cli/requirements.py3.Darwin.txt @@ -43,7 +43,7 @@ azure-mgmt-devtestlabs==4.0.0 azure-mgmt-dns==2.1.0 azure-mgmt-eventgrid==2.2.0 azure-mgmt-eventhub==4.0.0 -azure-mgmt-hdinsight==1.4.0 +azure-mgmt-hdinsight==1.5.1 azure-mgmt-imagebuilder==0.4.0 azure-mgmt-iotcentral==3.0.0 azure-mgmt-iothub==0.12.0 diff --git a/src/azure-cli/requirements.py3.Linux.txt b/src/azure-cli/requirements.py3.Linux.txt index ef49c9e3f63..d7c4f698601 100644 --- a/src/azure-cli/requirements.py3.Linux.txt +++ b/src/azure-cli/requirements.py3.Linux.txt @@ -43,7 +43,7 @@ azure-mgmt-devtestlabs==4.0.0 azure-mgmt-dns==2.1.0 azure-mgmt-eventgrid==2.2.0 azure-mgmt-eventhub==4.0.0 -azure-mgmt-hdinsight==1.4.0 +azure-mgmt-hdinsight==1.5.1 azure-mgmt-imagebuilder==0.4.0 azure-mgmt-iotcentral==3.0.0 azure-mgmt-iothub==0.12.0 diff --git a/src/azure-cli/requirements.py3.windows.txt b/src/azure-cli/requirements.py3.windows.txt index f0e03b0c91d..9c6ae931f48 100644 --- a/src/azure-cli/requirements.py3.windows.txt +++ b/src/azure-cli/requirements.py3.windows.txt @@ -42,7 +42,7 @@ azure-mgmt-devtestlabs==4.0.0 azure-mgmt-dns==2.1.0 azure-mgmt-eventgrid==2.2.0 azure-mgmt-eventhub==4.0.0 -azure-mgmt-hdinsight==1.4.0 +azure-mgmt-hdinsight==1.5.1 azure-mgmt-imagebuilder==0.4.0 azure-mgmt-iotcentral==3.0.0 azure-mgmt-iothub==0.12.0 diff --git a/src/azure-cli/setup.py b/src/azure-cli/setup.py index 4b79a65fcc6..76fa33fec81 100644 --- a/src/azure-cli/setup.py +++ b/src/azure-cli/setup.py @@ -85,7 +85,7 @@ 'azure-mgmt-dns~=2.1', 'azure-mgmt-eventgrid~=2.2', 'azure-mgmt-eventhub~=4.0.0', - 'azure-mgmt-hdinsight~=1.4.0', + 'azure-mgmt-hdinsight~=1.5.1', 'azure-mgmt-imagebuilder~=0.4.0', 'azure-mgmt-iotcentral~=3.0.0', 'azure-mgmt-iothub~=0.12.0', From 901afe71c442029b7b8eb202de6a30bd0e13662a Mon Sep 17 00:00:00 2001 From: Zhenyu Zhou Date: Mon, 29 Jun 2020 19:51:15 +0800 Subject: [PATCH 3/5] Change location back 2 eastus2 --- .../hdinsight/tests/latest/test_hdinsight_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py index a53b2efd53a..a9f5918c06b 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py @@ -12,7 +12,7 @@ class HDInsightClusterTests(ScenarioTest): - location = 'southcentralus' + location = 'eastus2' # Uses 'rg' kwarg @ResourceGroupPreparer(name_prefix='hdicli-', location=location, random_name_length=12) From bdacf8a2ae1f2726b05192a9cb47e93b041c78fe Mon Sep 17 00:00:00 2001 From: Zhenyu Zhou Date: Tue, 30 Jun 2020 11:25:53 +0800 Subject: [PATCH 4/5] Update parameter name and help doc --- src/azure-cli/azure/cli/command_modules/hdinsight/_help.py | 3 ++- src/azure-cli/azure/cli/command_modules/hdinsight/_params.py | 4 ++-- .../latest/recordings/test_hdinsight_virtual_machine.yaml | 4 ++-- .../hdinsight/tests/latest/test_hdinsight_commands.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/_help.py b/src/azure-cli/azure/cli/command_modules/hdinsight/_help.py index 97d33b47ef0..561ca0bb71f 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/_help.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/_help.py @@ -5,6 +5,7 @@ # -------------------------------------------------------------------------------------------- from knack.help_files import helps # pylint: disable=unused-import + # pylint: disable=line-too-long, too-many-lines helps['hdinsight'] = """ @@ -182,7 +183,7 @@ examples: - name: Restart the specific hosts of the specified HDInsight cluster. text: |- - az hdinsight host restart --resource-group MyResourceGroup --cluster-name MyCluster --hosts hostname + az hdinsight host restart --resource-group MyResourceGroup --cluster-name MyCluster --host-names hostname1 hostname2 """ helps['hdinsight wait'] = """ diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/_params.py b/src/azure-cli/azure/cli/command_modules/hdinsight/_params.py index 68f646ee33d..78f2883c006 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/_params.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/_params.py @@ -34,7 +34,6 @@ def load_arguments(self, _): # cluster with self.argument_context('hdinsight') as c: - # Cluster c.argument('cluster_name', arg_type=name_type, completer=get_resource_name_completion_list('Microsoft.HDInsight/clusters'), @@ -242,4 +241,5 @@ def load_arguments(self, _): c.argument('cluster_name', options_list=['--cluster-name'], completer=get_resource_name_completion_list('Microsoft.HDInsight/clusters'), help='The name of the cluster.') - c.argument('hosts', nargs="+", help='The host name of the cluster.') + c.argument('hosts', options_list=['--host-names'], nargs='+', + help='A space-delimited list of host names that need to be restarted.') diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/recordings/test_hdinsight_virtual_machine.yaml b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/recordings/test_hdinsight_virtual_machine.yaml index ade9404e198..27c35a3725c 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/recordings/test_hdinsight_virtual_machine.yaml +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/recordings/test_hdinsight_virtual_machine.yaml @@ -2069,7 +2069,7 @@ interactions: Content-Type: - application/json; charset=utf-8 ParameterSetName: - - --resource-group --cluster-name --hosts --yes + - --resource-group --cluster-name --host-names --yes User-Agent: - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 @@ -2114,7 +2114,7 @@ interactions: Connection: - keep-alive ParameterSetName: - - --resource-group --cluster-name --hosts --yes + - --resource-group --cluster-name --host-names --yes User-Agent: - python/3.7.4 (Windows-10-10.0.18362-SP0) msrest/0.6.11 msrest_azure/0.6.3 azure-mgmt-hdinsight/1.5.1 Azure-SDK-For-Python AZURECLI/2.8.0 diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py index a9f5918c06b..fbcd82c78e1 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py @@ -330,7 +330,7 @@ def test_hdinsight_virtual_machine(self, storage_account_info): break self.kwargs['target_host'] = target_host # restart host of the cluster - self.cmd('az hdinsight host restart --resource-group {rg} --cluster-name {cluster} --hosts {target_host} --yes') + self.cmd('az hdinsight host restart --resource-group {rg} --cluster-name {cluster} --host-names {target_host} --yes') def _create_hdinsight_cluster(self, *additional_create_arguments): self.kwargs.update({ From 3dbfb7e97df4d85f66170bf3d9378ae2af51238c Mon Sep 17 00:00:00 2001 From: Zhenyu Zhou Date: Wed, 1 Jul 2020 12:09:39 +0800 Subject: [PATCH 5/5] Removed support_no_wait --- src/azure-cli/azure/cli/command_modules/hdinsight/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure-cli/azure/cli/command_modules/hdinsight/commands.py b/src/azure-cli/azure/cli/command_modules/hdinsight/commands.py index a676ec18f44..6ef2c2bb089 100644 --- a/src/azure-cli/azure/cli/command_modules/hdinsight/commands.py +++ b/src/azure-cli/azure/cli/command_modules/hdinsight/commands.py @@ -103,4 +103,4 @@ def load_command_table(self, _): with self.command_group('hdinsight host', hdinsight_virtual_machines_sdk, client_factory=cf_hdinsight_virtual_machines) as g: g.command('list', 'list_hosts') - g.command('restart', 'restart_hosts', confirmation=True, supports_no_wait=True) + g.command('restart', 'restart_hosts', confirmation=True)