From 97c60feb1c303f8e69d720b29b58b3d90d714f37 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Sat, 11 Jul 2020 09:24:20 -0700 Subject: [PATCH 01/10] Use eventhub mgmt client to create test eventhubs namespace and resources --- sdk/eventhub/azure-eventhub/conftest.py | 204 -------------- .../azure-eventhub/dev_requirements.txt | 2 +- sdk/eventhub/azure-eventhub/tests/conftest.py | 258 ++++++++++++++++++ .../livetest/asynctests/test_auth_async.py | 13 +- .../asynctests/test_negative_async.py | 6 +- .../tests/livetest/synctests/test_auth.py | 18 +- .../tests/livetest/synctests/test_negative.py | 9 +- sdk/eventhub/tests.yml | 7 +- 8 files changed, 281 insertions(+), 236 deletions(-) delete mode 100644 sdk/eventhub/azure-eventhub/conftest.py create mode 100644 sdk/eventhub/azure-eventhub/tests/conftest.py diff --git a/sdk/eventhub/azure-eventhub/conftest.py b/sdk/eventhub/azure-eventhub/conftest.py deleted file mode 100644 index 6377d9a062d6..000000000000 --- a/sdk/eventhub/azure-eventhub/conftest.py +++ /dev/null @@ -1,204 +0,0 @@ -#------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -#-------------------------------------------------------------------------- - -import os -import pytest -import logging -import sys -import uuid -import warnings -from logging.handlers import RotatingFileHandler - -# Ignore async tests for Python < 3.5 -collect_ignore = [] -if sys.version_info < (3, 5): - collect_ignore.append("tests/livetest/asynctests") - collect_ignore.append("tests/unittest/asynctests") - collect_ignore.append("features") - collect_ignore.append("samples/async_samples") - collect_ignore.append("examples/async_examples") - -from azure.servicebus._control_client import ServiceBusService, EventHub -from azure.eventhub import EventHubProducerClient -import uamqp -from uamqp import authentication - -PARTITION_COUNT = 2 -CONN_STR = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}" - -def pytest_addoption(parser): - parser.addoption( - "--sleep", action="store", default="True", help="sleep on reconnect test: True or False" - ) - - -@pytest.fixture -def sleep(request): - sleep = request.config.getoption("--sleep") - return sleep.lower() in ('true', 'yes', '1', 'y') - - -def get_logger(filename, level=logging.INFO): - azure_logger = logging.getLogger("azure.eventhub") - azure_logger.setLevel(level) - uamqp_logger = logging.getLogger("uamqp") - uamqp_logger.setLevel(logging.INFO) - - formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') - console_handler = logging.StreamHandler(stream=sys.stdout) - console_handler.setFormatter(formatter) - if not azure_logger.handlers: - azure_logger.addHandler(console_handler) - if not uamqp_logger.handlers: - uamqp_logger.addHandler(console_handler) - - if filename: - file_handler = RotatingFileHandler(filename, maxBytes=5*1024*1024, backupCount=2) - file_handler.setFormatter(formatter) - azure_logger.addHandler(file_handler) - - return azure_logger - - -log = get_logger(None, logging.DEBUG) - - -def create_eventhub(eventhub_config, client=None): - hub_name = str(uuid.uuid4()) - hub_value = EventHub(partition_count=PARTITION_COUNT) - client = client or ServiceBusService( - service_namespace=eventhub_config['namespace'], - shared_access_key_name=eventhub_config['key_name'], - shared_access_key_value=eventhub_config['access_key']) - if client.create_event_hub(hub_name, hub=hub_value, fail_on_exist=True): - return hub_name - raise ValueError("EventHub creation failed.") - - -def cleanup_eventhub(eventhub_config, hub_name, client=None): - client = client or ServiceBusService( - service_namespace=eventhub_config['namespace'], - shared_access_key_name=eventhub_config['key_name'], - shared_access_key_value=eventhub_config['access_key']) - client.delete_event_hub(hub_name) - - -@pytest.fixture(scope="session") -def live_eventhub_config(): - try: - config = {} - config['hostname'] = os.environ['EVENT_HUB_HOSTNAME'] - config['event_hub'] = os.environ['EVENT_HUB_NAME'] - config['key_name'] = os.environ['EVENT_HUB_SAS_POLICY'] - config['access_key'] = os.environ['EVENT_HUB_SAS_KEY'] - config['namespace'] = os.environ['EVENT_HUB_NAMESPACE'] - config['consumer_group'] = "$Default" - config['partition'] = "0" - config['connection_str'] = CONN_STR - except KeyError: - pytest.skip("Live EventHub configuration not found.") - else: - return config - - -@pytest.fixture() -def live_eventhub(live_eventhub_config): # pylint: disable=redefined-outer-name - client = ServiceBusService( - service_namespace=live_eventhub_config['namespace'], - shared_access_key_name=live_eventhub_config['key_name'], - shared_access_key_value=live_eventhub_config['access_key']) - try: - hub_name = create_eventhub(live_eventhub_config, client=client) - print("Created EventHub {}".format(hub_name)) - live_eventhub_config['event_hub'] = hub_name - yield live_eventhub_config - finally: - try: - cleanup_eventhub(live_eventhub_config, hub_name, client=client) - print("Deleted EventHub {}".format(hub_name)) - except: - warnings.warn(UserWarning("eventhub teardown failed")) - - -@pytest.fixture() -def connection_str(live_eventhub): - return CONN_STR.format( - live_eventhub['hostname'], - live_eventhub['key_name'], - live_eventhub['access_key'], - live_eventhub['event_hub']) - - -@pytest.fixture() -def invalid_hostname(live_eventhub_config): - return CONN_STR.format( - "invalid123.servicebus.windows.net", - live_eventhub_config['key_name'], - live_eventhub_config['access_key'], - live_eventhub_config['event_hub']) - - -@pytest.fixture() -def invalid_key(live_eventhub_config): - return CONN_STR.format( - live_eventhub_config['hostname'], - live_eventhub_config['key_name'], - "invalid", - live_eventhub_config['event_hub']) - - -@pytest.fixture() -def invalid_policy(live_eventhub_config): - return CONN_STR.format( - live_eventhub_config['hostname'], - "invalid", - live_eventhub_config['access_key'], - live_eventhub_config['event_hub']) - - -@pytest.fixture() -def aad_credential(): - try: - return os.environ['AZURE_CLIENT_ID'], os.environ['AZURE_CLIENT_SECRET'], os.environ['AZURE_TENANT_ID'] - except KeyError: - pytest.skip('No Azure Active Directory credential found') - - -@pytest.fixture() -def connstr_receivers(connection_str, live_eventhub_config): - partitions = [str(i) for i in range(PARTITION_COUNT)] - receivers = [] - for p in partitions: - uri = "sb://{}/{}".format(live_eventhub_config['hostname'], live_eventhub_config['event_hub']) - sas_auth = authentication.SASTokenAuth.from_shared_access_key( - uri, live_eventhub_config['key_name'], live_eventhub_config['access_key']) - - source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format( - live_eventhub_config['hostname'], - live_eventhub_config['event_hub'], - live_eventhub_config['consumer_group'], - p) - receiver = uamqp.ReceiveClient(source, auth=sas_auth, debug=False, timeout=0, prefetch=500) - receiver.open() - receivers.append(receiver) - yield connection_str, receivers - for r in receivers: - r.close() - - -@pytest.fixture() -def connstr_senders(connection_str): - client = EventHubProducerClient.from_connection_string(connection_str) - partitions = client.get_partition_ids() - - senders = [] - for p in partitions: - sender = client._create_producer(partition_id=p) - senders.append(sender) - yield connection_str, senders - for s in senders: - s.close() - client.close() diff --git a/sdk/eventhub/azure-eventhub/dev_requirements.txt b/sdk/eventhub/azure-eventhub/dev_requirements.txt index b7dc2798b680..cdcaebafb03b 100644 --- a/sdk/eventhub/azure-eventhub/dev_requirements.txt +++ b/sdk/eventhub/azure-eventhub/dev_requirements.txt @@ -1,7 +1,7 @@ -e ../../../tools/azure-sdk-tools ../../core/azure-core -e ../../identity/azure-identity --e ../../servicebus/azure-servicebus +-e ../azure-mgmt-eventhub aiohttp>=3.0; python_version >= '3.5' docutils>=0.14 pygments>=2.2.0 diff --git a/sdk/eventhub/azure-eventhub/tests/conftest.py b/sdk/eventhub/azure-eventhub/tests/conftest.py new file mode 100644 index 000000000000..57d8e907d9d1 --- /dev/null +++ b/sdk/eventhub/azure-eventhub/tests/conftest.py @@ -0,0 +1,258 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import os + +import pytest +import logging +import sys +import uuid +import warnings +from logging.handlers import RotatingFileHandler + +from azure.identity import EnvironmentCredential +from azure.mgmt.resource import ResourceManagementClient +from azure.mgmt.eventhub import EventHubManagementClient +from azure.mgmt.eventhub.models import Eventhub +from azure.eventhub import EventHubProducerClient +import uamqp +from uamqp import authentication + +# Ignore async tests for Python < 3.5 +collect_ignore = [] +if sys.version_info < (3, 5): + collect_ignore.append("tests/livetest/asynctests") + collect_ignore.append("tests/unittest/asynctests") + collect_ignore.append("features") + collect_ignore.append("samples/async_samples") + collect_ignore.append("examples/async_examples") + +PARTITION_COUNT = 2 +CONN_STR = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}" +RES_GROUP_PREFIX = "eh-res-group" +NAMESPACE_PREFIX = "eh-ns" +EVENTHUB_PREFIX = "eh" +EVENTHUB_DEFAULT_AUTH_RULE_NAME = 'RootManageSharedAccessKey' +LOCATION = "westus" + +TENANT_ID = os.environ["AZURE_TENANT_ID"] +CLIENT_ID = os.environ["AZURE_CLIENT_ID"] +SECRET = os.environ["AZURE_CLIENT_SECRET"] +SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] + +def pytest_addoption(parser): + parser.addoption( + "--sleep", action="store", default="True", help="sleep on reconnect test: True or False" + ) + + +@pytest.fixture +def sleep(request): + sleep = request.config.getoption("--sleep") + return sleep.lower() in ('true', 'yes', '1', 'y') + + +def get_logger(filename, level=logging.INFO): + azure_logger = logging.getLogger("azure.eventhub") + azure_logger.setLevel(level) + uamqp_logger = logging.getLogger("uamqp") + uamqp_logger.setLevel(logging.INFO) + + formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') + console_handler = logging.StreamHandler(stream=sys.stdout) + console_handler.setFormatter(formatter) + if not azure_logger.handlers: + azure_logger.addHandler(console_handler) + if not uamqp_logger.handlers: + uamqp_logger.addHandler(console_handler) + + if filename: + file_handler = RotatingFileHandler(filename, maxBytes=5*1024*1024, backupCount=2) + file_handler.setFormatter(formatter) + azure_logger.addHandler(file_handler) + + return azure_logger + + +log = get_logger(None, logging.DEBUG) + + +def create_eventhub(eventhub_config): + client = eventhub_config["management_client"] + resource_group = eventhub_config["resource_group"] + namespace = eventhub_config["namespace"] + eventhub_name = eventhub_config["eventhub"] + hub_value = Eventhub(partition_count=PARTITION_COUNT) + client.operations.create_or_update(resource_group, namespace, eventhub_name, hub_value) + + +def cleanup_eventhub(eventhub_config): + client = eventhub_config["management_client"] + resource_group = eventhub_config["resource_group"] + namespace = eventhub_config["namespace"] + eventhub_name = eventhub_config["eventhub"] + client.operations.delete(resource_group, namespace, eventhub_name) + + +@pytest.fixture(scope="session") +def resource_group(): + resource_client = ResourceManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) + resource_group_name = RES_GROUP_PREFIX + str(uuid.uuid4()) + try: + rg = resource_client.resource_groups.create_or_update( + resource_group_name, {"location": LOCATION} + ) + yield rg + finally: + try: + resource_client.resource_groups.begin_delete(resource_group_name) + except: + warnings.warn(UserWarning("resource group teardown failed")) + + +@pytest.fixture(scope="session") +def eventhub_namespace(resource_group): + resource_client = EventHubManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) + namespace_name = NAMESPACE_PREFIX + str(uuid.uuid4()) + try: + namespace = resource_client.namespaces.begin_create_or_update( + resource_group.name, namespace_name, {"location": LOCATION} + ).result() + key = resource_client.namespaces.list_keys(resource_group.name, namespace_name, EVENTHUB_DEFAULT_AUTH_RULE_NAME) + connection_string = key.primary_connection_string + key_name = key.key_name + primary_key = key.primary_key + yield namespace.name, connection_string, key_name, primary_key + finally: + try: + resource_client.namespaces.begin_delete(resource_group.name, namespace_name).wait() + except: + warnings.warn(UserWarning("eventhub namespace teardown failed")) + + +@pytest.fixture() +def live_eventhub(resource_group, eventhub_namespace): # pylint: disable=redefined-outer-name + resource_client = EventHubManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) + eventhub_name = EVENTHUB_PREFIX + str(uuid.uuid4()) + eventhub_ns_name, connection_string, key_name, primary_key = eventhub_namespace + try: + eventhub = resource_client.event_hubs.create_or_update( + resource_group.name, eventhub_ns_name, eventhub_name, {"partition_count": PARTITION_COUNT} + ) + live_eventhub_config = { + 'resource_group': resource_group.name, + 'hostname': "{}.servicebus.windows.net".format(eventhub_ns_name), + 'key_name': key_name, + 'access_key': primary_key, + 'namespace': eventhub_ns_name, + 'event_hub': eventhub.name, + 'consumer_group': '$Default', + 'partition': '0', + 'connection_str': connection_string + ";EntityPath="+eventhub.name + } + yield live_eventhub_config + finally: + try: + resource_client.event_hubs.delete(resource_group.name, eventhub_ns_name, eventhub_name) + except: + warnings.warn(UserWarning("eventhub teardown failed")) + + +@pytest.fixture() +def connection_str(live_eventhub): + return CONN_STR.format( + live_eventhub['hostname'], + live_eventhub['key_name'], + live_eventhub['access_key'], + live_eventhub['event_hub']) + + +@pytest.fixture() +def invalid_hostname(live_eventhub): + return CONN_STR.format( + "invalid123.servicebus.windows.net", + live_eventhub['key_name'], + live_eventhub['access_key'], + live_eventhub['event_hub']) + + +@pytest.fixture() +def invalid_key(live_eventhub): + return CONN_STR.format( + live_eventhub['hostname'], + live_eventhub['key_name'], + "invalid", + live_eventhub['event_hub']) + + +@pytest.fixture() +def invalid_policy(live_eventhub): + return CONN_STR.format( + live_eventhub['hostname'], + "invalid", + live_eventhub['access_key'], + live_eventhub['event_hub']) + + +@pytest.fixture(scope="session") +def aad_credential(): + try: + return os.environ['AZURE_CLIENT_ID'], os.environ['AZURE_CLIENT_SECRET'], os.environ['AZURE_TENANT_ID'] + except KeyError: + pytest.skip('No Azure Active Directory credential found') + +@pytest.fixture(scope="session") +def aad_credential_test_eh(): + try: + config = {} + config['hostname'] = os.environ['EVENT_HUB_HOSTNAME'] + config['event_hub'] = os.environ['EVENT_HUB_NAME'] + config['key_name'] = os.environ['EVENT_HUB_SAS_POLICY'] + config['access_key'] = os.environ['EVENT_HUB_SAS_KEY'] + config['namespace'] = os.environ['EVENT_HUB_NAMESPACE'] + except KeyError: + pytest.skip("Live EventHub configuration not found.") + else: + return config + + +@pytest.fixture() +def connstr_receivers(live_eventhub): + connection_str = live_eventhub["connection_str"] + partitions = [str(i) for i in range(PARTITION_COUNT)] + receivers = [] + for p in partitions: + uri = "sb://{}/{}".format(live_eventhub['hostname'], live_eventhub['event_hub']) + sas_auth = authentication.SASTokenAuth.from_shared_access_key( + uri, live_eventhub['key_name'], live_eventhub['access_key']) + + source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format( + live_eventhub['hostname'], + live_eventhub['event_hub'], + live_eventhub['consumer_group'], + p) + receiver = uamqp.ReceiveClient(source, auth=sas_auth, debug=False, timeout=0, prefetch=500) + receiver.open() + receivers.append(receiver) + yield connection_str, receivers + for r in receivers: + r.close() + + +@pytest.fixture() +def connstr_senders(live_eventhub): + connection_str = live_eventhub["connection_str"] + client = EventHubProducerClient.from_connection_string(connection_str) + partitions = client.get_partition_ids() + + senders = [] + for p in partitions: + sender = client._create_producer(partition_id=p) + senders.append(sender) + yield connection_str, senders + for s in senders: + s.close() + client.close() diff --git a/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_auth_async.py b/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_auth_async.py index 96842a2298fa..ed15c198e369 100644 --- a/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_auth_async.py +++ b/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_auth_async.py @@ -13,7 +13,7 @@ @pytest.mark.liveTest @pytest.mark.asyncio -async def test_client_secret_credential_async(aad_credential, live_eventhub): +async def test_client_secret_credential_async(aad_credential, aad_credential_test_eh): try: from azure.identity.aio import EnvironmentCredential except ImportError: @@ -21,12 +21,12 @@ async def test_client_secret_credential_async(aad_credential, live_eventhub): return credential = EnvironmentCredential() - producer_client = EventHubProducerClient(fully_qualified_namespace=live_eventhub['hostname'], - eventhub_name=live_eventhub['event_hub'], + producer_client = EventHubProducerClient(fully_qualified_namespace=aad_credential_test_eh['hostname'], + eventhub_name=aad_credential_test_eh['event_hub'], credential=credential, user_agent='customized information') - consumer_client = EventHubConsumerClient(fully_qualified_namespace=live_eventhub['hostname'], - eventhub_name=live_eventhub['event_hub'], + consumer_client = EventHubConsumerClient(fully_qualified_namespace=aad_credential_test_eh['hostname'], + eventhub_name=aad_credential_test_eh['event_hub'], consumer_group='$default', credential=credential, user_agent='customized information') @@ -46,5 +46,4 @@ def on_event(partition_context, event): await asyncio.sleep(13) await task assert on_event.called is True - assert on_event.partition_id == "0" - assert list(on_event.event.body)[0] == 'A single message'.encode('utf-8') + diff --git a/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_negative_async.py b/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_negative_async.py index 09ef3a14be4d..eb1fc0beb663 100644 --- a/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_negative_async.py +++ b/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_negative_async.py @@ -40,11 +40,11 @@ async def test_send_with_invalid_hostname_async(invalid_hostname, connstr_receiv ["hostname", "key_name", "access_key", "event_hub", "partition"]) @pytest.mark.liveTest @pytest.mark.asyncio -async def test_receive_with_invalid_param_async(live_eventhub_config, invalid_place): - eventhub_config = live_eventhub_config.copy() +async def test_receive_with_invalid_param_async(live_eventhub, invalid_place): + eventhub_config = live_eventhub.copy() if invalid_place != "partition": eventhub_config[invalid_place] = "invalid " + invalid_place - conn_str = live_eventhub_config["connection_str"].format( + conn_str = live_eventhub["connection_str"].format( eventhub_config['hostname'], eventhub_config['key_name'], eventhub_config['access_key'], diff --git a/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_auth.py b/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_auth.py index aba361ed645f..0e8ae48e8222 100644 --- a/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_auth.py +++ b/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_auth.py @@ -11,18 +11,15 @@ @pytest.mark.liveTest -def test_client_secret_credential(aad_credential, live_eventhub): - try: - from azure.identity import EnvironmentCredential - except: - pytest.skip("No azure identity library") +def test_client_secret_credential(aad_credential, aad_credential_test_eh): + from azure.identity import EnvironmentCredential credential = EnvironmentCredential() - producer_client = EventHubProducerClient(fully_qualified_namespace=live_eventhub['hostname'], - eventhub_name=live_eventhub['event_hub'], + producer_client = EventHubProducerClient(fully_qualified_namespace=aad_credential_test_eh['hostname'], + eventhub_name=aad_credential_test_eh['event_hub'], credential=credential, user_agent='customized information') - consumer_client = EventHubConsumerClient(fully_qualified_namespace=live_eventhub['hostname'], - eventhub_name=live_eventhub['event_hub'], + consumer_client = EventHubConsumerClient(fully_qualified_namespace=aad_credential_test_eh['hostname'], + eventhub_name=aad_credential_test_eh['event_hub'], consumer_group='$default', credential=credential, user_agent='customized information') @@ -47,5 +44,4 @@ def on_event(partition_context, event): worker.join() assert on_event.called is True - assert on_event.partition_id == "0" - assert list(on_event.event.body)[0] == 'A single message'.encode('utf-8') + diff --git a/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_negative.py b/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_negative.py index 5228f1aa4148..d3b775381796 100644 --- a/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_negative.py +++ b/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_negative.py @@ -50,13 +50,8 @@ def on_event(partition_context, event): @pytest.mark.liveTest -def test_send_batch_with_invalid_key(live_eventhub): - conn_str = live_eventhub["connection_str"].format( - live_eventhub['hostname'], - live_eventhub['key_name'], - 'invalid', - live_eventhub['event_hub']) - client = EventHubProducerClient.from_connection_string(conn_str) +def test_send_batch_with_invalid_key(invalid_key): + client = EventHubProducerClient.from_connection_string(invalid_key) try: with pytest.raises(ConnectError): batch = EventDataBatch() diff --git a/sdk/eventhub/tests.yml b/sdk/eventhub/tests.yml index 8b3513f01bad..66d2b248adc2 100644 --- a/sdk/eventhub/tests.yml +++ b/sdk/eventhub/tests.yml @@ -30,6 +30,7 @@ jobs: EVENT_HUB_NAMESPACE: $(python-eh-livetest-event-hub-namespace) IOTHUB_CONNECTION_STR: $(python-eh-livetest-event-hub-iothub-connection-str) IOTHUB_DEVICE: $(python-eh-livetest-event-hub-iothub-device) - AZURE_CLIENT_ID: $(python-eh-livetest-event-hub-aad-client-id) - AZURE_TENANT_ID: $(python-eh-livetest-event-hub-aad-tenant-id) - AZURE_CLIENT_SECRET: $(python-eh-livetest-event-hub-aad-secret) + AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id) + AZURE_SUBSCRIPTION_ID: $(azure-subscription-id) + AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret) + AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id) From 19c9e74734213379bc7dfcd6aca4818cec322512 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Sat, 11 Jul 2020 10:24:15 -0700 Subject: [PATCH 02/10] Use azure-mgmt-resource preview as dev req --- sdk/eventhub/azure-eventhub/dev_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/eventhub/azure-eventhub/dev_requirements.txt b/sdk/eventhub/azure-eventhub/dev_requirements.txt index cdcaebafb03b..d068041b7bd9 100644 --- a/sdk/eventhub/azure-eventhub/dev_requirements.txt +++ b/sdk/eventhub/azure-eventhub/dev_requirements.txt @@ -2,6 +2,7 @@ ../../core/azure-core -e ../../identity/azure-identity -e ../azure-mgmt-eventhub +-e ../../resources/azure-mgmt-resource aiohttp>=3.0; python_version >= '3.5' docutils>=0.14 pygments>=2.2.0 From 0c607caf48be0f00138a7fde60190dfcb1ae3a68 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Sat, 11 Jul 2020 10:24:27 -0700 Subject: [PATCH 03/10] remove unused code --- sdk/eventhub/azure-eventhub/tests/conftest.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/sdk/eventhub/azure-eventhub/tests/conftest.py b/sdk/eventhub/azure-eventhub/tests/conftest.py index 57d8e907d9d1..f5ca69a4ccef 100644 --- a/sdk/eventhub/azure-eventhub/tests/conftest.py +++ b/sdk/eventhub/azure-eventhub/tests/conftest.py @@ -80,23 +80,6 @@ def get_logger(filename, level=logging.INFO): log = get_logger(None, logging.DEBUG) -def create_eventhub(eventhub_config): - client = eventhub_config["management_client"] - resource_group = eventhub_config["resource_group"] - namespace = eventhub_config["namespace"] - eventhub_name = eventhub_config["eventhub"] - hub_value = Eventhub(partition_count=PARTITION_COUNT) - client.operations.create_or_update(resource_group, namespace, eventhub_name, hub_value) - - -def cleanup_eventhub(eventhub_config): - client = eventhub_config["management_client"] - resource_group = eventhub_config["resource_group"] - namespace = eventhub_config["namespace"] - eventhub_name = eventhub_config["eventhub"] - client.operations.delete(resource_group, namespace, eventhub_name) - - @pytest.fixture(scope="session") def resource_group(): resource_client = ResourceManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) From f2fdaae3abd634810985b7fa01a540176565a127 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Mon, 13 Jul 2020 11:09:08 -0700 Subject: [PATCH 04/10] Move conftest.py to project root directory --- .../azure-eventhub/{tests => }/conftest.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) rename sdk/eventhub/azure-eventhub/{tests => }/conftest.py (96%) diff --git a/sdk/eventhub/azure-eventhub/tests/conftest.py b/sdk/eventhub/azure-eventhub/conftest.py similarity index 96% rename from sdk/eventhub/azure-eventhub/tests/conftest.py rename to sdk/eventhub/azure-eventhub/conftest.py index f5ca69a4ccef..088df4479530 100644 --- a/sdk/eventhub/azure-eventhub/tests/conftest.py +++ b/sdk/eventhub/azure-eventhub/conftest.py @@ -3,12 +3,10 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. #-------------------------------------------------------------------------- - +import sys import os - import pytest import logging -import sys import uuid import warnings from logging.handlers import RotatingFileHandler @@ -16,10 +14,10 @@ from azure.identity import EnvironmentCredential from azure.mgmt.resource import ResourceManagementClient from azure.mgmt.eventhub import EventHubManagementClient -from azure.mgmt.eventhub.models import Eventhub from azure.eventhub import EventHubProducerClient -import uamqp -from uamqp import authentication +from uamqp import ReceiveClient +from uamqp.authentication import SASTokenAuth + # Ignore async tests for Python < 3.5 collect_ignore = [] @@ -28,8 +26,6 @@ collect_ignore.append("tests/unittest/asynctests") collect_ignore.append("features") collect_ignore.append("samples/async_samples") - collect_ignore.append("examples/async_examples") - PARTITION_COUNT = 2 CONN_STR = "Endpoint=sb://{}/;SharedAccessKeyName={};SharedAccessKey={};EntityPath={}" RES_GROUP_PREFIX = "eh-res-group" @@ -209,7 +205,7 @@ def connstr_receivers(live_eventhub): receivers = [] for p in partitions: uri = "sb://{}/{}".format(live_eventhub['hostname'], live_eventhub['event_hub']) - sas_auth = authentication.SASTokenAuth.from_shared_access_key( + sas_auth = SASTokenAuth.from_shared_access_key( uri, live_eventhub['key_name'], live_eventhub['access_key']) source = "amqps://{}/{}/ConsumerGroups/{}/Partitions/{}".format( @@ -217,7 +213,7 @@ def connstr_receivers(live_eventhub): live_eventhub['event_hub'], live_eventhub['consumer_group'], p) - receiver = uamqp.ReceiveClient(source, auth=sas_auth, debug=False, timeout=0, prefetch=500) + receiver = ReceiveClient(source, auth=sas_auth, debug=False, timeout=0, prefetch=500) receiver.open() receivers.append(receiver) yield connection_str, receivers From 035a87743aad443169315d27815137f34c6376e0 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Mon, 13 Jul 2020 11:09:44 -0700 Subject: [PATCH 05/10] Add variable python-eh-livetest-event-hub-subscription-id --- sdk/eventhub/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/eventhub/tests.yml b/sdk/eventhub/tests.yml index 66d2b248adc2..d4f7da2e5f17 100644 --- a/sdk/eventhub/tests.yml +++ b/sdk/eventhub/tests.yml @@ -30,7 +30,7 @@ jobs: EVENT_HUB_NAMESPACE: $(python-eh-livetest-event-hub-namespace) IOTHUB_CONNECTION_STR: $(python-eh-livetest-event-hub-iothub-connection-str) IOTHUB_DEVICE: $(python-eh-livetest-event-hub-iothub-device) - AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id) - AZURE_SUBSCRIPTION_ID: $(azure-subscription-id) - AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret) - AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id) + AZURE_CLIENT_ID: $(python-eh-livetest-event-hub-aad-client-id) + AZURE_TENANT_ID: $(python-eh-livetest-event-hub-aad-tenant-id) + AZURE_CLIENT_SECRET: $(python-eh-livetest-event-hub-aad-secret) + AZURE_SUBSCRIPTION_ID: $(python-eh-livetest-event-hub-subscription-id) From 275cf608ca5bd93c95db9e057f698c4b6633c6c4 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Mon, 13 Jul 2020 12:43:45 -0700 Subject: [PATCH 06/10] Use created eventhub for AAD test --- sdk/eventhub/azure-eventhub/conftest.py | 51 +++++-------------- .../livetest/asynctests/test_auth_async.py | 20 +++----- .../tests/livetest/synctests/test_auth.py | 15 +++--- 3 files changed, 30 insertions(+), 56 deletions(-) diff --git a/sdk/eventhub/azure-eventhub/conftest.py b/sdk/eventhub/azure-eventhub/conftest.py index 088df4479530..fe468a8c524b 100644 --- a/sdk/eventhub/azure-eventhub/conftest.py +++ b/sdk/eventhub/azure-eventhub/conftest.py @@ -34,10 +34,6 @@ EVENTHUB_DEFAULT_AUTH_RULE_NAME = 'RootManageSharedAccessKey' LOCATION = "westus" -TENANT_ID = os.environ["AZURE_TENANT_ID"] -CLIENT_ID = os.environ["AZURE_CLIENT_ID"] -SECRET = os.environ["AZURE_CLIENT_SECRET"] -SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] def pytest_addoption(parser): parser.addoption( @@ -78,22 +74,24 @@ def get_logger(filename, level=logging.INFO): @pytest.fixture(scope="session") def resource_group(): - resource_client = ResourceManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) - resource_group_name = RES_GROUP_PREFIX + str(uuid.uuid4()) + SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] + resource_client = ResourceManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) + resource_group_name = RES_GROUP_PREFIX + str(uuid.uuid4()) + try: + rg = resource_client.resource_groups.create_or_update( + resource_group_name, {"location": LOCATION} + ) + yield rg + finally: try: - rg = resource_client.resource_groups.create_or_update( - resource_group_name, {"location": LOCATION} - ) - yield rg - finally: - try: - resource_client.resource_groups.begin_delete(resource_group_name) - except: - warnings.warn(UserWarning("resource group teardown failed")) + resource_client.resource_groups.begin_delete(resource_group_name) + except: + warnings.warn(UserWarning("resource group teardown failed")) @pytest.fixture(scope="session") def eventhub_namespace(resource_group): + SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] resource_client = EventHubManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) namespace_name = NAMESPACE_PREFIX + str(uuid.uuid4()) try: @@ -114,6 +112,7 @@ def eventhub_namespace(resource_group): @pytest.fixture() def live_eventhub(resource_group, eventhub_namespace): # pylint: disable=redefined-outer-name + SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] resource_client = EventHubManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) eventhub_name = EVENTHUB_PREFIX + str(uuid.uuid4()) eventhub_ns_name, connection_string, key_name, primary_key = eventhub_namespace @@ -176,28 +175,6 @@ def invalid_policy(live_eventhub): live_eventhub['event_hub']) -@pytest.fixture(scope="session") -def aad_credential(): - try: - return os.environ['AZURE_CLIENT_ID'], os.environ['AZURE_CLIENT_SECRET'], os.environ['AZURE_TENANT_ID'] - except KeyError: - pytest.skip('No Azure Active Directory credential found') - -@pytest.fixture(scope="session") -def aad_credential_test_eh(): - try: - config = {} - config['hostname'] = os.environ['EVENT_HUB_HOSTNAME'] - config['event_hub'] = os.environ['EVENT_HUB_NAME'] - config['key_name'] = os.environ['EVENT_HUB_SAS_POLICY'] - config['access_key'] = os.environ['EVENT_HUB_SAS_KEY'] - config['namespace'] = os.environ['EVENT_HUB_NAMESPACE'] - except KeyError: - pytest.skip("Live EventHub configuration not found.") - else: - return config - - @pytest.fixture() def connstr_receivers(live_eventhub): connection_str = live_eventhub["connection_str"] diff --git a/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_auth_async.py b/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_auth_async.py index ed15c198e369..1a055feacfde 100644 --- a/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_auth_async.py +++ b/sdk/eventhub/azure-eventhub/tests/livetest/asynctests/test_auth_async.py @@ -7,26 +7,21 @@ import pytest import asyncio +from azure.identity.aio import EnvironmentCredential from azure.eventhub import EventData from azure.eventhub.aio import EventHubConsumerClient, EventHubProducerClient @pytest.mark.liveTest @pytest.mark.asyncio -async def test_client_secret_credential_async(aad_credential, aad_credential_test_eh): - try: - from azure.identity.aio import EnvironmentCredential - except ImportError: - pytest.skip("No azure identity library") - return - +async def test_client_secret_credential_async(live_eventhub): credential = EnvironmentCredential() - producer_client = EventHubProducerClient(fully_qualified_namespace=aad_credential_test_eh['hostname'], - eventhub_name=aad_credential_test_eh['event_hub'], + producer_client = EventHubProducerClient(fully_qualified_namespace=live_eventhub['hostname'], + eventhub_name=live_eventhub['event_hub'], credential=credential, user_agent='customized information') - consumer_client = EventHubConsumerClient(fully_qualified_namespace=aad_credential_test_eh['hostname'], - eventhub_name=aad_credential_test_eh['event_hub'], + consumer_client = EventHubConsumerClient(fully_qualified_namespace=live_eventhub['hostname'], + eventhub_name=live_eventhub['event_hub'], consumer_group='$default', credential=credential, user_agent='customized information') @@ -46,4 +41,5 @@ def on_event(partition_context, event): await asyncio.sleep(13) await task assert on_event.called is True - + assert on_event.partition_id == "0" + assert list(on_event.event.body)[0] == 'A single message'.encode('utf-8') diff --git a/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_auth.py b/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_auth.py index 0e8ae48e8222..3959164d902d 100644 --- a/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_auth.py +++ b/sdk/eventhub/azure-eventhub/tests/livetest/synctests/test_auth.py @@ -7,19 +7,19 @@ import time import threading +from azure.identity import EnvironmentCredential from azure.eventhub import EventData, EventHubProducerClient, EventHubConsumerClient @pytest.mark.liveTest -def test_client_secret_credential(aad_credential, aad_credential_test_eh): - from azure.identity import EnvironmentCredential +def test_client_secret_credential(live_eventhub): credential = EnvironmentCredential() - producer_client = EventHubProducerClient(fully_qualified_namespace=aad_credential_test_eh['hostname'], - eventhub_name=aad_credential_test_eh['event_hub'], + producer_client = EventHubProducerClient(fully_qualified_namespace=live_eventhub['hostname'], + eventhub_name=live_eventhub['event_hub'], credential=credential, user_agent='customized information') - consumer_client = EventHubConsumerClient(fully_qualified_namespace=aad_credential_test_eh['hostname'], - eventhub_name=aad_credential_test_eh['event_hub'], + consumer_client = EventHubConsumerClient(fully_qualified_namespace=live_eventhub['hostname'], + eventhub_name=live_eventhub['event_hub'], consumer_group='$default', credential=credential, user_agent='customized information') @@ -44,4 +44,5 @@ def on_event(partition_context, event): worker.join() assert on_event.called is True - + assert on_event.partition_id == "0" + assert list(on_event.event.body)[0] == 'A single message'.encode('utf-8') From d55532e015bb497e85b0d669ba932c87ee8233d9 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Mon, 13 Jul 2020 13:49:38 -0700 Subject: [PATCH 07/10] skip CI test if no SUBSCRIPTION_ID is defined --- sdk/eventhub/azure-eventhub/conftest.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/sdk/eventhub/azure-eventhub/conftest.py b/sdk/eventhub/azure-eventhub/conftest.py index fe468a8c524b..2ee89fee84e9 100644 --- a/sdk/eventhub/azure-eventhub/conftest.py +++ b/sdk/eventhub/azure-eventhub/conftest.py @@ -74,7 +74,11 @@ def get_logger(filename, level=logging.INFO): @pytest.fixture(scope="session") def resource_group(): - SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] + try: + SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] + except KeyError: + pytest.skip('AZURE_SUBSCRIPTION_ID defined') + return resource_client = ResourceManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) resource_group_name = RES_GROUP_PREFIX + str(uuid.uuid4()) try: @@ -91,7 +95,11 @@ def resource_group(): @pytest.fixture(scope="session") def eventhub_namespace(resource_group): - SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] + try: + SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] + except KeyError: + pytest.skip('AZURE_SUBSCRIPTION_ID defined') + return resource_client = EventHubManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) namespace_name = NAMESPACE_PREFIX + str(uuid.uuid4()) try: @@ -112,7 +120,11 @@ def eventhub_namespace(resource_group): @pytest.fixture() def live_eventhub(resource_group, eventhub_namespace): # pylint: disable=redefined-outer-name - SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] + try: + SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] + except KeyError: + pytest.skip('AZURE_SUBSCRIPTION_ID defined') + return resource_client = EventHubManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) eventhub_name = EVENTHUB_PREFIX + str(uuid.uuid4()) eventhub_ns_name, connection_string, key_name, primary_key = eventhub_namespace From 4ef90b8ca3dc49f6de39f1372d7681eee7c4b3b5 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Fri, 17 Jul 2020 17:46:38 -0700 Subject: [PATCH 08/10] Update skip reason Co-authored-by: Adam Ling (MSFT) <47871814+yunhaoling@users.noreply.github.com> --- sdk/eventhub/azure-eventhub/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/eventhub/azure-eventhub/conftest.py b/sdk/eventhub/azure-eventhub/conftest.py index 2ee89fee84e9..41a52d7790ed 100644 --- a/sdk/eventhub/azure-eventhub/conftest.py +++ b/sdk/eventhub/azure-eventhub/conftest.py @@ -77,7 +77,7 @@ def resource_group(): try: SUBSCRIPTION_ID = os.environ["AZURE_SUBSCRIPTION_ID"] except KeyError: - pytest.skip('AZURE_SUBSCRIPTION_ID defined') + pytest.skip('AZURE_SUBSCRIPTION_ID undefined') return resource_client = ResourceManagementClient(EnvironmentCredential(), SUBSCRIPTION_ID) resource_group_name = RES_GROUP_PREFIX + str(uuid.uuid4()) From b1819f42222224fcf30b635198ab1e1bcce3eab5 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Tue, 21 Jul 2020 16:21:37 -0700 Subject: [PATCH 09/10] Add eventhub_preparer.py --- .../azure-eventhub/tests/eventhub_preparer.py | 309 ++++++++++++++++++ 1 file changed, 309 insertions(+) create mode 100644 sdk/eventhub/azure-eventhub/tests/eventhub_preparer.py diff --git a/sdk/eventhub/azure-eventhub/tests/eventhub_preparer.py b/sdk/eventhub/azure-eventhub/tests/eventhub_preparer.py new file mode 100644 index 000000000000..907278197a2d --- /dev/null +++ b/sdk/eventhub/azure-eventhub/tests/eventhub_preparer.py @@ -0,0 +1,309 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +# This EventHubs resource preparer is for future use after the tests are all compatible with +# The resource preparer. + +import functools + +from azure.mgmt.eventhub import EventHubManagementClient +from azure.mgmt.eventhub.models import Eventhub, AccessRights + +from azure_devtools.scenario_tests.exceptions import AzureTestError + +from devtools_testutils import ( + ResourceGroupPreparer, AzureMgmtPreparer, FakeResource +) + +from devtools_testutils.resource_testcase import RESOURCE_GROUP_PARAM + +EVENTHUB_DEFAULT_AUTH_RULE_NAME = 'RootManageSharedAccessKey' +EVENTHUB_NAMESPACE_PARAM = 'eventhub_namespace' +EVENTHUB_PARAM = 'eventhub' +EVENTHUB_AUTHORIZATION_RULE_PARAM = 'eventhub_authorization_rule' + +# Service Bus Namespace Preparer and its shorthand decorator +class EventHubNamespacePreparer(AzureMgmtPreparer): + def __init__(self, + name_prefix='', + use_cache=False, + sku='Standard', location='westus', + parameter_name=EVENTHUB_NAMESPACE_PARAM, + resource_group_parameter_name=RESOURCE_GROUP_PARAM, + disable_recording=True, playback_fake_resource=None, + client_kwargs=None, random_name_enabled=True): + super(EventHubNamespacePreparer, self).__init__(name_prefix, 24, + random_name_enabled=random_name_enabled, + disable_recording=disable_recording, + playback_fake_resource=playback_fake_resource, + client_kwargs=client_kwargs) + self.location = location + self.sku = sku + self.resource_group_parameter_name = resource_group_parameter_name + self.parameter_name = parameter_name + self.connection_string = '' + if random_name_enabled: + self.resource_moniker = self.name_prefix + "ehname" + + self.set_cache(use_cache, sku, location) + + def create_resource(self, name, **kwargs): + if self.is_live: + self.client = self.create_mgmt_client(EventHubManagementClient) # type: EventHubManagementClient + group = self._get_resource_group(**kwargs) + namespace_async_operation = self.client.namespaces.begin_create_or_update( + group.name, + name, + { + 'sku': {'name': self.sku}, + 'location': self.location, + } + ) + self.resource = namespace_async_operation.result() + + key = self.client.namespaces.list_keys(group.name, name, EVENTHUB_DEFAULT_AUTH_RULE_NAME) + self.connection_string = key.primary_connection_string + self.key_name = key.key_name + self.primary_key = key.primary_key + + self.test_class_instance.scrubber.register_name_pair( + name, + self.resource_moniker + ) + else: + self.resource = FakeResource(name=name, id=name) + self.connection_string = 'Endpoint=sb://{}.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='.format(name) + self.key_name = EVENTHUB_DEFAULT_AUTH_RULE_NAME + self.primary_key = 'ZmFrZV9hY29jdW50X2tleQ==' + return { + self.parameter_name: self.resource, + '{}_connection_string'.format(self.parameter_name): self.connection_string, + '{}_key_name'.format(self.parameter_name): self.key_name, + '{}_primary_key'.format(self.parameter_name): self.primary_key, + '{}_management_client'.format(self.parameter_name): self.client + } + + def remove_resource(self, name, **kwargs): + if self.is_live: + group = self._get_resource_group(**kwargs) + self.client.namespaces.begin_delete(group.name, name).wait() + + def _get_resource_group(self, **kwargs): + try: + return kwargs.get(self.resource_group_parameter_name) + except KeyError: + template = 'To create an event hub a resource group is required. Please add ' \ + 'decorator @{} in front of this event hub preparer.' + raise AzureTestError(template.format(ResourceGroupPreparer.__name__)) + + +# Shared base class for event hub sub-resources that require a namespace and RG to exist. +class _EventHubChildResourcePreparer(AzureMgmtPreparer): + def __init__(self, + name_prefix='', + resource_group_parameter_name=RESOURCE_GROUP_PARAM, + eventhub_namespace_parameter_name=EVENTHUB_NAMESPACE_PARAM, + disable_recording=True, playback_fake_resource=None, + client_kwargs=None, random_name_enabled=True): + super(_EventHubChildResourcePreparer, self).__init__(name_prefix, 24, + random_name_enabled=random_name_enabled, + disable_recording=disable_recording, + playback_fake_resource=playback_fake_resource, + client_kwargs=client_kwargs) + self.resource_group_parameter_name = resource_group_parameter_name + self.eventhub_namespace_parameter_name = eventhub_namespace_parameter_name + + def _get_resource_group(self, **kwargs): + try: + return kwargs.get(self.resource_group_parameter_name) + except KeyError: + template = 'To create this event hub child resource event hub a resource group is required. Please add ' \ + 'decorator @{} in front of this event hub preparer.' + raise AzureTestError(template.format(ResourceGroupPreparer.__name__)) + + def _get_namespace(self, **kwargs): + try: + return kwargs.get(self.eventhub_namespace_parameter_name) + except KeyError: + template = 'To create this event hub child resource a event hub namespace is required. Please add ' \ + 'decorator @{} in front of this event hub preparer.' + raise AzureTestError(template.format(EventHubNamespacePreparer.__name__)) + + +class EventHubPreparer(_EventHubChildResourcePreparer): + def __init__(self, + name_prefix='', + use_cache=False, + parameter_name=EVENTHUB_PARAM, + resource_group_parameter_name=RESOURCE_GROUP_PARAM, + eventhub_namespace_parameter_name=EVENTHUB_NAMESPACE_PARAM, + disable_recording=True, playback_fake_resource=None, + client_kwargs=None, random_name_enabled=True): + super(EventHubPreparer, self).__init__(name_prefix, + random_name_enabled=random_name_enabled, + resource_group_parameter_name=resource_group_parameter_name, + eventhub_namespace_parameter_name=eventhub_namespace_parameter_name, + disable_recording=disable_recording, + playback_fake_resource=playback_fake_resource, + client_kwargs=client_kwargs) + self.parameter_name = parameter_name + if random_name_enabled: + self.resource_moniker = self.name_prefix + "eventhub" + self.set_cache(use_cache) + + def create_resource(self, name, **kwargs): + if self.is_live: + self.client = self.create_mgmt_client(EventHubManagementClient) + group = self._get_resource_group(**kwargs) + namespace = self._get_namespace(**kwargs) + self.resource = self.client.event_hubs.create_or_update( + group.name, + namespace.name, + name, + {} + ) + else: + self.resource = FakeResource(name=name, id=name) + return { + self.parameter_name: self.resource, + } + + def remove_resource(self, name, **kwargs): + if self.is_live: + group = self._get_resource_group(**kwargs) + namespace = self._get_namespace(**kwargs) + self.client.event_hubs.delete(group.name, namespace.name, name, polling=False) + + +class EventHubNamespaceAuthorizationRulePreparer(_EventHubChildResourcePreparer): + def __init__(self, + name_prefix='', + use_cache=False, + access_rights=[AccessRights.manage, AccessRights.send, AccessRights.listen], + parameter_name=EVENTHUB_AUTHORIZATION_RULE_PARAM, + resource_group_parameter_name=RESOURCE_GROUP_PARAM, + eventhub_namespace_parameter_name=EVENTHUB_NAMESPACE_PARAM, + disable_recording=True, playback_fake_resource=None, + client_kwargs=None, random_name_enabled=True): + super(EventHubNamespaceAuthorizationRulePreparer, self).__init__(name_prefix, + random_name_enabled=random_name_enabled, + resource_group_parameter_name=resource_group_parameter_name, + eventhub_namespace_parameter_name=eventhub_namespace_parameter_name, + disable_recording=disable_recording, + playback_fake_resource=playback_fake_resource, + client_kwargs=client_kwargs) + self.parameter_name = parameter_name + self.access_rights = access_rights + if random_name_enabled: + self.resource_moniker = self.name_prefix + "sbnameauth" + self.set_cache(use_cache, access_rights) + + def create_resource(self, name, **kwargs): + if self.is_live: + self.client = self.create_mgmt_client(EventHubManagementClient) + group = self._get_resource_group(**kwargs) + namespace = self._get_namespace(**kwargs) + self.resource = self.client.namespaces.create_or_update_authorization_rule( + group.name, + namespace.name, + name, + self.access_rights + ) + + key = self.client.namespaces.list_keys(group.name, namespace.name, name) + connection_string = key.primary_connection_string + + self.test_class_instance.scrubber.register_name_pair( + name, + self.resource_moniker + ) + else: + self.resource = FakeResource(name=name, id=name) + connection_string = 'https://microsoft.com' + return { + self.parameter_name: self.resource, + '{}_connection_string'.format(self.parameter_name): connection_string, + } + + def remove_resource(self, name, **kwargs): + if self.is_live: + group = self._get_resource_group(**kwargs) + namespace = self._get_namespace(**kwargs) + self.client.namespaces.delete_authorization_rule(group.name, namespace.name, name, polling=False) + + +class EventHubAuthorizationRulePreparer(_EventHubChildResourcePreparer): + def __init__(self, + name_prefix='', + use_cache=False, + access_rights=[AccessRights.manage, AccessRights.send, AccessRights.listen], + parameter_name=EVENTHUB_AUTHORIZATION_RULE_PARAM, + resource_group_parameter_name=RESOURCE_GROUP_PARAM, + eventhub_namespace_parameter_name=EVENTHUB_NAMESPACE_PARAM, + eventhub_parameter_name=EVENTHUB_PARAM, + disable_recording=True, playback_fake_resource=None, + client_kwargs=None, random_name_enabled=True): + super(EventHubAuthorizationRulePreparer, self).__init__(name_prefix, + random_name_enabled=random_name_enabled, + resource_group_parameter_name=resource_group_parameter_name, + eventhub_namespace_parameter_name=eventhub_namespace_parameter_name, + disable_recording=disable_recording, + playback_fake_resource=playback_fake_resource, + client_kwargs=client_kwargs) + self.parameter_name = parameter_name + self.access_rights = access_rights + self.eventhub_parameter_name = eventhub_parameter_name + if random_name_enabled: + self.resource_moniker = self.name_prefix + "evnethubauth" + self.set_cache(use_cache, access_rights) + + def create_resource(self, name, **kwargs): + if self.is_live: + self.client = self.create_mgmt_client(EventHubManagementClient) + group = self._get_resource_group(**kwargs) + namespace = self._get_namespace(**kwargs) + eventhub = self._get_eventhub(**kwargs) + self.resource = self.client.event_hubs.create_or_update_authorization_rule( + group.name, + namespace.name, + eventhub.name, + name, + self.access_rights + ) + + key = self.client.event_hubs.list_keys(group.name, namespace.name, eventhub.name, name) + connection_string = key.primary_connection_string + + self.test_class_instance.scrubber.register_name_pair( + name, + self.resource_moniker + ) + else: + self.resource = FakeResource(name=name, id=name) + connection_string = 'https://microsoft.com' + return { + self.parameter_name: self.resource, + '{}_connection_string'.format(self.parameter_name): connection_string, + } + + def remove_resource(self, name, **kwargs): + if self.is_live: + group = self._get_resource_group(**kwargs) + namespace = self._get_namespace(**kwargs) + eventhub = self._get_eventhub(**kwargs) + self.client.event_hubs.delete_authorization_rule(group.name, namespace.name, eventhub.name, name, polling=False) + + def _get_eventhub(self, **kwargs): + try: + return kwargs.get(self.eventhub_parameter_name) + except KeyError: + template = 'To create this event hub authorization rule a event hub is required. Please add ' \ + 'decorator @{} in front of this event hub preparer.' + raise AzureTestError(template.format(EventHubPreparer.__name__)) + + +CachedEventHubNamespacePreparer = functools.partial(EventHubNamespacePreparer, use_cache=True) +CachedEventHubPreparer = functools.partial(EventHubPreparer, use_cache=True) From ff142625cc6a0e7a3a70728b7e5d6a4f3a7c31e9 Mon Sep 17 00:00:00 2001 From: Yijun Xie <48257664+YijunXieMS@users.noreply.github.com> Date: Wed, 22 Jul 2020 12:13:08 -0700 Subject: [PATCH 10/10] Remove scrubber --- .../azure-eventhub/tests/eventhub_preparer.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/sdk/eventhub/azure-eventhub/tests/eventhub_preparer.py b/sdk/eventhub/azure-eventhub/tests/eventhub_preparer.py index 907278197a2d..d9e99ed2c1ab 100644 --- a/sdk/eventhub/azure-eventhub/tests/eventhub_preparer.py +++ b/sdk/eventhub/azure-eventhub/tests/eventhub_preparer.py @@ -68,11 +68,6 @@ def create_resource(self, name, **kwargs): self.connection_string = key.primary_connection_string self.key_name = key.key_name self.primary_key = key.primary_key - - self.test_class_instance.scrubber.register_name_pair( - name, - self.resource_moniker - ) else: self.resource = FakeResource(name=name, id=name) self.connection_string = 'Endpoint=sb://{}.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='.format(name) @@ -215,11 +210,6 @@ def create_resource(self, name, **kwargs): key = self.client.namespaces.list_keys(group.name, namespace.name, name) connection_string = key.primary_connection_string - - self.test_class_instance.scrubber.register_name_pair( - name, - self.resource_moniker - ) else: self.resource = FakeResource(name=name, id=name) connection_string = 'https://microsoft.com' @@ -276,11 +266,6 @@ def create_resource(self, name, **kwargs): key = self.client.event_hubs.list_keys(group.name, namespace.name, eventhub.name, name) connection_string = key.primary_connection_string - - self.test_class_instance.scrubber.register_name_pair( - name, - self.resource_moniker - ) else: self.resource = FakeResource(name=name, id=name) connection_string = 'https://microsoft.com'