From c118f47adc6690381f5b03bbf306c1d04bcd78dc Mon Sep 17 00:00:00 2001 From: AsafMah Date: Tue, 25 Jul 2023 11:01:33 +0300 Subject: [PATCH 01/13] DRY in env code, maybe fix bug. --- .../azure/kusto/data/_cloud_settings.py | 4 +- .../azure/kusto/data/client_details.py | 5 ++- .../azure/kusto/data/env_utils.py | 37 +++++++++++++++++ .../tests/aio/test_async_token_providers.py | 37 ++++++++--------- azure-kusto-data/tests/e2e.py | 29 ++++++------- .../tests/test_token_providers.py | 41 ++++++++----------- azure-kusto-ingest/tests/e2e.py | 26 +++++------- 7 files changed, 99 insertions(+), 80 deletions(-) create mode 100644 azure-kusto-data/azure/kusto/data/env_utils.py diff --git a/azure-kusto-data/azure/kusto/data/_cloud_settings.py b/azure-kusto-data/azure/kusto/data/_cloud_settings.py index da5eb3a7..70c1c80a 100644 --- a/azure-kusto-data/azure/kusto/data/_cloud_settings.py +++ b/azure-kusto-data/azure/kusto/data/_cloud_settings.py @@ -1,5 +1,4 @@ import dataclasses -import os from threading import Lock from typing import Optional, Dict from urllib.parse import urljoin @@ -9,6 +8,7 @@ from azure.core.tracing.decorator import distributed_trace from azure.core.tracing import SpanKind +import env_utils from ._telemetry import Span, MonitoredActivity from .exceptions import KustoServiceError @@ -46,7 +46,7 @@ class CloudSettings: _cloud_cache_lock = Lock() DEFAULT_CLOUD = CloudInfo( - login_endpoint=os.environ.get(DEFAULT_AUTH_ENV_VAR_NAME, DEFAULT_PUBLIC_LOGIN_URL), + login_endpoint=env_utils.get_env(DEFAULT_AUTH_ENV_VAR_NAME, default=DEFAULT_PUBLIC_LOGIN_URL), login_mfa_required=False, kusto_client_app_id=DEFAULT_KUSTO_CLIENT_APP_ID, kusto_client_redirect_uri=DEFAULT_REDIRECT_URI, diff --git a/azure-kusto-data/azure/kusto/data/client_details.py b/azure-kusto-data/azure/kusto/data/client_details.py index cb380717..444c3d1e 100644 --- a/azure-kusto-data/azure/kusto/data/client_details.py +++ b/azure-kusto-data/azure/kusto/data/client_details.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from typing import List, Tuple, Optional +import env_utils from azure.kusto.data._version import VERSION NONE = "[none]" @@ -23,8 +24,8 @@ def default_script() -> str: @functools.lru_cache(maxsize=1) def get_user_from_env() -> str: - user = os.environ.get("USERNAME", None) - domain = os.environ.get("USERDOMAIN", None) + user = env_utils.get_env("USERNAME", optional=True) + domain = env_utils.get_env("USERDOMAIN", optional=True) if domain and user: user = domain + "\\" + user if user: diff --git a/azure-kusto-data/azure/kusto/data/env_utils.py b/azure-kusto-data/azure/kusto/data/env_utils.py new file mode 100644 index 00000000..48fa6c0c --- /dev/null +++ b/azure-kusto-data/azure/kusto/data/env_utils.py @@ -0,0 +1,37 @@ +import os + + +def get_env(*args, optional=False, default=None): + """Return the first environment variable that is defined.""" + for arg in args: + if arg in os.environ: + return os.environ[arg] + if optional or default: + return default + raise ValueError("No environment variables found: {}".format(args)) + + +def set_env(key, value): + """Set the environment variable.""" + os.environ[key] = value + + +def get_app_id(optional=False): + """Return the app id.""" + result = get_env("APP_ID", "AZURE_CLIENT_ID", optional=optional) + os.environ["AZURE_CLIENT_ID"] = result + return result + + +def get_auth_id(optional=False): + """Return the auth id.""" + result = get_env("AUTH_ID", "APP_AUTH_ID", "AZURE_TENANT_ID", optional=optional) + os.environ["AZURE_TENANT_ID"] = result + return result + + +def get_app_key(optional=False): + """Return the app key.""" + result = get_env("APP_KEY", "AZURE_CLIENT_SECRET", optional=optional) + os.environ["AZURE_CLIENT_SECRET"] = result + return result diff --git a/azure-kusto-data/tests/aio/test_async_token_providers.py b/azure-kusto-data/tests/aio/test_async_token_providers.py index 03ec6b46..2b0de9ea 100644 --- a/azure-kusto-data/tests/aio/test_async_token_providers.py +++ b/azure-kusto-data/tests/aio/test_async_token_providers.py @@ -5,6 +5,7 @@ import pytest from azure.identity.aio import ClientSecretCredential as AsyncClientSecretCredential +import env_utils from azure.kusto.data._decorators import aio_documented_by from azure.kusto.data._token_providers import * from .test_kusto_client import run_aio_tests @@ -162,8 +163,8 @@ async def test_msi_provider(self): print(" *** Skipped MSI Provider Test ***") return - user_msi_object_id = os.environ.get("MSI_OBJECT_ID") - user_msi_client_id = os.environ.get("MSI_CLIENT_ID") + user_msi_object_id = env_utils.get_env("MSI_OBJECT_ID") + user_msi_client_id = env_utils.get_env("MSI_CLIENT_ID") # system MSI with MsiTokenProvider(KUSTO_URI, is_async=True) as provider: @@ -189,9 +190,9 @@ async def test_msi_provider(self): @aio_documented_by(TokenProviderTests.test_user_pass_provider) @pytest.mark.asyncio async def test_user_pass_provider(self): - username = os.environ.get("USER_NAME") - password = os.environ.get("USER_PASS") - auth = os.environ.get("USER_AUTH_ID", "organizations") + username = env_utils.get_env("USER_NAME") + password = env_utils.get_env("USER_PASS") + auth = env_utils.get_env("USER_AUTH_ID", default="organizations") if username and password and auth: with UserPassTokenProvider(KUSTO_URI, auth, username, password, is_async=True) as provider: @@ -228,9 +229,9 @@ def callback(x, x2, x3): async def test_app_key_provider(self): # default details are for kusto-client-e2e-test-app # to run the test, get the key from Azure portal - app_id = os.environ.get("APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f") - auth_id = os.environ.get("APP_AUTH_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47") - app_key = os.environ.get("APP_KEY") + app_id = env_utils.get_app_id(optional=True) + auth_id = env_utils.get_auth_id(optional=True) + app_key = env_utils.get_app_key(optional=True) if app_id and app_key and auth_id: with ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key, is_async=True) as provider: @@ -248,11 +249,11 @@ async def test_app_key_provider(self): async def test_app_cert_provider(self): # default details are for kusto-client-e2e-test-app # to invoke the test download the certs from Azure Portal - cert_app_id = os.environ.get("CERT_APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f") - cert_auth = os.environ.get("CERT_AUTH", "72f988bf-86f1-41af-91ab-2d7cd011db47") - thumbprint = os.environ.get("CERT_THUMBPRINT") - public_cert_path = os.environ.get("PUBLIC_CERT_PATH") - pem_key_path = os.environ.get("CERT_PEM_KEY_PATH") + cert_app_id = env_utils.get_app_id(optional=True) + cert_auth = env_utils.get_auth_id(optional=True) + thumbprint = env_utils.get_env("CERT_THUMBPRINT", optional=True) + public_cert_path = env_utils.get_env("CERT_PUBLIC_CERT_PATH", optional=True) + pem_key_path = env_utils.get_env("CERT_PEM_KEY_PATH", optional=True) if pem_key_path and thumbprint and cert_app_id: with open(pem_key_path, "rb") as file: @@ -351,13 +352,9 @@ async def inner(): @aio_documented_by(TokenProviderTests.test_azure_identity_default_token_provider) @pytest.mark.asyncio async def test_azure_identity_token_provider(self): - app_id = os.environ.get("APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f") - os.environ["AZURE_CLIENT_ID"] = app_id - auth_id = os.environ.get("APP_AUTH_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47") - os.environ["AZURE_TENANT_ID"] = auth_id - app_key = os.environ.get("APP_KEY") - os.environ["AZURE_CLIENT_SECRET"] = app_key - + app_id = env_utils.get_app_id() + auth_id = env_utils.get_auth_id() + app_key = env_utils.get_app_key() with AzureIdentityTokenCredentialProvider(KUSTO_URI, is_async=True, credential=AsyncDefaultAzureCredential()) as provider: token = await provider.get_token_async() assert TokenProviderTests.get_token_value(token) is not None diff --git a/azure-kusto-data/tests/e2e.py b/azure-kusto-data/tests/e2e.py index 3639e687..35655a17 100644 --- a/azure-kusto-data/tests/e2e.py +++ b/azure-kusto-data/tests/e2e.py @@ -12,6 +12,7 @@ import pytest from azure.identity import DefaultAzureCredential +import env_utils from azure.kusto.data import KustoClient, KustoConnectionStringBuilder from azure.kusto.data._cloud_settings import CloudSettings, DEFAULT_DEV_KUSTO_SERVICE_RESOURCE_ID from azure.kusto.data._models import WellKnownDataSet @@ -108,23 +109,17 @@ def engine_kcsb_from_env(cls, app_insights=False, is_async=False) -> KustoConnec @classmethod def setup_class(cls): - cls.engine_cs = os.environ.get("ENGINE_CONNECTION_STRING") or "" - cls.ai_engine_cs = os.environ.get("APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING") or "" - cls.app_id = os.environ.get("APP_ID") - if cls.app_id: - os.environ["AZURE_CLIENT_ID"] = cls.app_id - cls.app_key = os.environ.get("APP_KEY") - if cls.app_key: - os.environ["AZURE_CLIENT_SECRET"] = cls.app_key - cls.auth_id = os.environ.get("AUTH_ID") - if cls.auth_id: - os.environ["AZURE_TENANT_ID"] = cls.auth_id - os.environ["AZURE_AUTHORITY_HOST"] = "login.microsoftonline.com" - cls.test_db = os.environ.get("TEST_DATABASE") - cls.ai_test_db = os.environ.get("APPLICATION_INSIGHTS_TEST_DATABASE") # name of e2e database could be changed - - if not all([cls.engine_cs, cls.test_db, cls.ai_engine_cs, cls.ai_test_db]): - pytest.skip("E2E environment is missing") + cls.engine_cs = env_utils.get_env("ENGINE_CONNECTION_STRING") + cls.ai_engine_cs = env_utils.get_env("APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING") + + cls.app_id = env_utils.get_app_id() + cls.auth_id = env_utils.get_auth_id() + cls.app_key = env_utils.get_app_key() + + env_utils.set_env("AZURE_AUTHORITY_HOST", "login.microsoftonline.com") + + cls.test_db = env_utils.get_env("TEST_DATABASE") + cls.ai_test_db = env_utils.get_env("APPLICATION_INSIGHTS_TEST_DATABASE") # name of e2e database could be changed # Init clients cls.streaming_test_table = "BigChunkus" diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index b0993213..f3acb808 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -1,12 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License -import os import unittest from threading import Thread from asgiref.sync import async_to_sync from azure.identity import ClientSecretCredential, DefaultAzureCredential +import env_utils from azure.kusto.data._token_providers import * KUSTO_URI = "https://sdkse2etest.eastus.kusto.windows.net" @@ -185,8 +185,8 @@ def test_msi_provider(): print(" *** Skipped MSI Provider Test ***") return - user_msi_object_id = os.environ.get("MSI_OBJECT_ID") - user_msi_client_id = os.environ.get("MSI_CLIENT_ID") + user_msi_object_id = env_utils.get_env("MSI_OBJECT_ID") + user_msi_client_id = env_utils.get_env("MSI_CLIENT_ID") # system MSI with MsiTokenProvider(KUSTO_URI) as provider: @@ -211,9 +211,9 @@ def test_msi_provider(): @staticmethod def test_user_pass_provider(): - username = os.environ.get("USER_NAME") - password = os.environ.get("USER_PASS") - auth = os.environ.get("USER_AUTH_ID", "organizations") + username = env_utils.get_env("USER_NAME") + password = env_utils.get_env("USER_PASS") + auth = env_utils.get_env("USER_AUTH_ID", default="organizations") if username and password and auth: with UserPassTokenProvider(KUSTO_URI, auth, username, password) as provider: @@ -250,7 +250,7 @@ def test_interactive_login(): print(" *** Skipped interactive login Test ***") return - auth_id = os.environ.get("APP_AUTH_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47") + auth_id = env_utils.get_auth_id() with InteractiveLoginTokenProvider(KUSTO_URI, auth_id) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -263,9 +263,9 @@ def test_interactive_login(): def test_app_key_provider(): # default details are for kusto-client-e2e-test-app # to run the test, get the key from Azure portal - app_id = os.environ.get("APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f") - auth_id = os.environ.get("APP_AUTH_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47") - app_key = os.environ.get("APP_KEY") + app_id = env_utils.get_app_id() + auth_id = env_utils.get_auth_id() + app_key = env_utils.get_app_key() if app_id and app_key and auth_id: with ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key) as provider: @@ -280,13 +280,11 @@ def test_app_key_provider(): @staticmethod def test_app_cert_provider(): - # default details are for kusto-client-e2e-test-app - # to run the test download the certs from Azure Portal - cert_app_id = os.environ.get("CERT_APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f") - cert_auth = os.environ.get("CERT_AUTH", "72f988bf-86f1-41af-91ab-2d7cd011db47") - thumbprint = os.environ.get("CERT_THUMBPRINT") - public_cert_path = os.environ.get("PUBLIC_CERT_PATH") - pem_key_path = os.environ.get("CERT_PEM_KEY_PATH") + cert_app_id = env_utils.get_app_id(optional=True) + cert_auth = env_utils.get_auth_id(optional=True) + thumbprint = env_utils.get_env("CERT_THUMBPRINT", optional=True) + public_cert_path = env_utils.get_env("CERT_PUBLIC_CERT_PATH", optional=True) + pem_key_path = env_utils.get_env("CERT_PEM_KEY_PATH", optional=True) if pem_key_path and thumbprint and cert_app_id: with open(pem_key_path, "rb") as file: @@ -361,12 +359,9 @@ def test_cloud_mfa_on(): @staticmethod def test_azure_identity_default_token_provider(): - app_id = os.environ.get("APP_ID", "b699d721-4f6f-4320-bc9a-88d578dfe68f") - os.environ["AZURE_CLIENT_ID"] = app_id - auth_id = os.environ.get("APP_AUTH_ID", "72f988bf-86f1-41af-91ab-2d7cd011db47") - os.environ["AZURE_TENANT_ID"] = auth_id - app_key = os.environ.get("APP_KEY") - os.environ["AZURE_CLIENT_SECRET"] = app_key + app_id = env_utils.get_app_id() + auth_id = env_utils.get_auth_id() + app_key = env_utils.get_app_key() with AzureIdentityTokenCredentialProvider(KUSTO_URI, credential=DefaultAzureCredential()) as provider: token = provider.get_token() diff --git a/azure-kusto-ingest/tests/e2e.py b/azure-kusto-ingest/tests/e2e.py index cf29a8d4..1cf083c3 100644 --- a/azure-kusto-ingest/tests/e2e.py +++ b/azure-kusto-ingest/tests/e2e.py @@ -13,6 +13,7 @@ from azure.identity import DefaultAzureCredential from typing import Optional, ClassVar +import env_utils from azure.kusto.data import KustoClient, KustoConnectionStringBuilder from azure.kusto.data._token_providers import AsyncDefaultAzureCredential from azure.kusto.data.aio import KustoClient as AsyncKustoClient @@ -168,22 +169,15 @@ def dm_kcsb_from_env(cls) -> KustoConnectionStringBuilder: @classmethod def setup_class(cls): # DM CS can be composed from engine CS - cls.engine_cs = os.environ.get("ENGINE_CONNECTION_STRING") or "" - cls.dm_cs = os.environ.get("DM_CONNECTION_STRING") or cls.engine_cs.replace("//", "//ingest-") - cls.app_id = os.environ.get("APP_ID") - if cls.app_id: - os.environ["AZURE_CLIENT_ID"] = cls.app_id - cls.app_key = os.environ.get("APP_KEY") - if cls.app_key: - os.environ["AZURE_CLIENT_SECRET"] = cls.app_key - cls.auth_id = os.environ.get("AUTH_ID") - if cls.auth_id: - os.environ["AZURE_TENANT_ID"] = cls.auth_id - cls.test_db = os.environ.get("TEST_DATABASE") - cls.test_blob = os.environ.get("TEST_BLOB") - - if not all([cls.engine_cs, cls.dm_cs, cls.test_db]): - pytest.skip("E2E environment is missing") + cls.engine_cs = env_utils.get_env("ENGINE_CONNECTION_STRING") + cls.ai_engine_cs = env_utils.get_env("APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING") + + cls.app_id = env_utils.get_app_id() + cls.auth_id = env_utils.get_auth_id() + cls.app_key = env_utils.get_app_key() + + cls.test_db = env_utils.get_env("TEST_DATABASE") + cls.test_blob = env_utils.get_env("TEST_BLOB", optional=True) # Init clients python_version = "_".join([str(v) for v in sys.version_info[:3]]) From c875a16f89daf23acfd33529b91a8c338be999d9 Mon Sep 17 00:00:00 2001 From: AsafMah Date: Tue, 25 Jul 2023 11:13:40 +0300 Subject: [PATCH 02/13] DRY in env code, maybe fix bug. --- .../azure/kusto/data/_cloud_settings.py | 4 +-- .../azure/kusto/data/client_details.py | 6 ++-- .../tests/aio/test_async_token_providers.py | 36 +++++++++---------- azure-kusto-data/tests/e2e.py | 18 +++++----- .../tests/test_token_providers.py | 36 +++++++++---------- azure-kusto-ingest/tests/e2e.py | 16 ++++----- 6 files changed, 57 insertions(+), 59 deletions(-) diff --git a/azure-kusto-data/azure/kusto/data/_cloud_settings.py b/azure-kusto-data/azure/kusto/data/_cloud_settings.py index 70c1c80a..0a47ec80 100644 --- a/azure-kusto-data/azure/kusto/data/_cloud_settings.py +++ b/azure-kusto-data/azure/kusto/data/_cloud_settings.py @@ -8,7 +8,7 @@ from azure.core.tracing.decorator import distributed_trace from azure.core.tracing import SpanKind -import env_utils +from .env_utils import get_env from ._telemetry import Span, MonitoredActivity from .exceptions import KustoServiceError @@ -46,7 +46,7 @@ class CloudSettings: _cloud_cache_lock = Lock() DEFAULT_CLOUD = CloudInfo( - login_endpoint=env_utils.get_env(DEFAULT_AUTH_ENV_VAR_NAME, default=DEFAULT_PUBLIC_LOGIN_URL), + login_endpoint=get_env(DEFAULT_AUTH_ENV_VAR_NAME, default=DEFAULT_PUBLIC_LOGIN_URL), login_mfa_required=False, kusto_client_app_id=DEFAULT_KUSTO_CLIENT_APP_ID, kusto_client_redirect_uri=DEFAULT_REDIRECT_URI, diff --git a/azure-kusto-data/azure/kusto/data/client_details.py b/azure-kusto-data/azure/kusto/data/client_details.py index 444c3d1e..e643350d 100644 --- a/azure-kusto-data/azure/kusto/data/client_details.py +++ b/azure-kusto-data/azure/kusto/data/client_details.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from typing import List, Tuple, Optional -import env_utils +from .env_utils import get_env from azure.kusto.data._version import VERSION NONE = "[none]" @@ -24,8 +24,8 @@ def default_script() -> str: @functools.lru_cache(maxsize=1) def get_user_from_env() -> str: - user = env_utils.get_env("USERNAME", optional=True) - domain = env_utils.get_env("USERDOMAIN", optional=True) + user = get_env("USERNAME", optional=True) + domain = get_env("USERDOMAIN", optional=True) if domain and user: user = domain + "\\" + user if user: diff --git a/azure-kusto-data/tests/aio/test_async_token_providers.py b/azure-kusto-data/tests/aio/test_async_token_providers.py index 2b0de9ea..b98610f9 100644 --- a/azure-kusto-data/tests/aio/test_async_token_providers.py +++ b/azure-kusto-data/tests/aio/test_async_token_providers.py @@ -1,13 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License -import os - import pytest from azure.identity.aio import ClientSecretCredential as AsyncClientSecretCredential -import env_utils from azure.kusto.data._decorators import aio_documented_by from azure.kusto.data._token_providers import * +from azure.kusto.data.env_utils import get_env, get_app_id, get_auth_id, get_app_key from .test_kusto_client import run_aio_tests from ..test_token_providers import KUSTO_URI, TOKEN_VALUE, TEST_AZ_AUTH, TEST_MSI_AUTH, TEST_DEVICE_AUTH, TokenProviderTests, MockProvider @@ -163,8 +161,8 @@ async def test_msi_provider(self): print(" *** Skipped MSI Provider Test ***") return - user_msi_object_id = env_utils.get_env("MSI_OBJECT_ID") - user_msi_client_id = env_utils.get_env("MSI_CLIENT_ID") + user_msi_object_id = get_env("MSI_OBJECT_ID") + user_msi_client_id = get_env("MSI_CLIENT_ID") # system MSI with MsiTokenProvider(KUSTO_URI, is_async=True) as provider: @@ -190,9 +188,9 @@ async def test_msi_provider(self): @aio_documented_by(TokenProviderTests.test_user_pass_provider) @pytest.mark.asyncio async def test_user_pass_provider(self): - username = env_utils.get_env("USER_NAME") - password = env_utils.get_env("USER_PASS") - auth = env_utils.get_env("USER_AUTH_ID", default="organizations") + username = get_env("USER_NAME") + password = get_env("USER_PASS") + auth = get_env("USER_AUTH_ID", default="organizations") if username and password and auth: with UserPassTokenProvider(KUSTO_URI, auth, username, password, is_async=True) as provider: @@ -229,9 +227,9 @@ def callback(x, x2, x3): async def test_app_key_provider(self): # default details are for kusto-client-e2e-test-app # to run the test, get the key from Azure portal - app_id = env_utils.get_app_id(optional=True) - auth_id = env_utils.get_auth_id(optional=True) - app_key = env_utils.get_app_key(optional=True) + app_id = get_app_id(optional=True) + auth_id = get_auth_id(optional=True) + app_key = get_app_key(optional=True) if app_id and app_key and auth_id: with ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key, is_async=True) as provider: @@ -249,11 +247,11 @@ async def test_app_key_provider(self): async def test_app_cert_provider(self): # default details are for kusto-client-e2e-test-app # to invoke the test download the certs from Azure Portal - cert_app_id = env_utils.get_app_id(optional=True) - cert_auth = env_utils.get_auth_id(optional=True) - thumbprint = env_utils.get_env("CERT_THUMBPRINT", optional=True) - public_cert_path = env_utils.get_env("CERT_PUBLIC_CERT_PATH", optional=True) - pem_key_path = env_utils.get_env("CERT_PEM_KEY_PATH", optional=True) + cert_app_id = get_app_id(optional=True) + cert_auth = get_auth_id(optional=True) + thumbprint = get_env("CERT_THUMBPRINT", optional=True) + public_cert_path = get_env("CERT_PUBLIC_CERT_PATH", optional=True) + pem_key_path = get_env("CERT_PEM_KEY_PATH", optional=True) if pem_key_path and thumbprint and cert_app_id: with open(pem_key_path, "rb") as file: @@ -352,9 +350,9 @@ async def inner(): @aio_documented_by(TokenProviderTests.test_azure_identity_default_token_provider) @pytest.mark.asyncio async def test_azure_identity_token_provider(self): - app_id = env_utils.get_app_id() - auth_id = env_utils.get_auth_id() - app_key = env_utils.get_app_key() + app_id = get_app_id() + auth_id = get_auth_id() + app_key = get_app_key() with AzureIdentityTokenCredentialProvider(KUSTO_URI, is_async=True, credential=AsyncDefaultAzureCredential()) as provider: token = await provider.get_token_async() assert TokenProviderTests.get_token_value(token) is not None diff --git a/azure-kusto-data/tests/e2e.py b/azure-kusto-data/tests/e2e.py index 35655a17..1828f213 100644 --- a/azure-kusto-data/tests/e2e.py +++ b/azure-kusto-data/tests/e2e.py @@ -12,7 +12,7 @@ import pytest from azure.identity import DefaultAzureCredential -import env_utils +from azure.kusto.data.env_utils import get_env, get_app_id, get_auth_id, get_app_key, set_env from azure.kusto.data import KustoClient, KustoConnectionStringBuilder from azure.kusto.data._cloud_settings import CloudSettings, DEFAULT_DEV_KUSTO_SERVICE_RESOURCE_ID from azure.kusto.data._models import WellKnownDataSet @@ -109,17 +109,17 @@ def engine_kcsb_from_env(cls, app_insights=False, is_async=False) -> KustoConnec @classmethod def setup_class(cls): - cls.engine_cs = env_utils.get_env("ENGINE_CONNECTION_STRING") - cls.ai_engine_cs = env_utils.get_env("APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING") + cls.engine_cs = get_env("ENGINE_CONNECTION_STRING") + cls.ai_engine_cs = get_env("APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING") - cls.app_id = env_utils.get_app_id() - cls.auth_id = env_utils.get_auth_id() - cls.app_key = env_utils.get_app_key() + cls.app_id = get_app_id() + cls.auth_id = get_auth_id() + cls.app_key = get_app_key() - env_utils.set_env("AZURE_AUTHORITY_HOST", "login.microsoftonline.com") + set_env("AZURE_AUTHORITY_HOST", "login.microsoftonline.com") - cls.test_db = env_utils.get_env("TEST_DATABASE") - cls.ai_test_db = env_utils.get_env("APPLICATION_INSIGHTS_TEST_DATABASE") # name of e2e database could be changed + cls.test_db = get_env("TEST_DATABASE") + cls.ai_test_db = get_env("APPLICATION_INSIGHTS_TEST_DATABASE") # name of e2e database could be changed # Init clients cls.streaming_test_table = "BigChunkus" diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index f3acb808..33009179 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -6,7 +6,7 @@ from asgiref.sync import async_to_sync from azure.identity import ClientSecretCredential, DefaultAzureCredential -import env_utils +from azure.kusto.data.env_utils import get_env, get_app_id, get_auth_id, get_app_key from azure.kusto.data._token_providers import * KUSTO_URI = "https://sdkse2etest.eastus.kusto.windows.net" @@ -185,8 +185,8 @@ def test_msi_provider(): print(" *** Skipped MSI Provider Test ***") return - user_msi_object_id = env_utils.get_env("MSI_OBJECT_ID") - user_msi_client_id = env_utils.get_env("MSI_CLIENT_ID") + user_msi_object_id = get_env("MSI_OBJECT_ID") + user_msi_client_id = get_env("MSI_CLIENT_ID") # system MSI with MsiTokenProvider(KUSTO_URI) as provider: @@ -211,9 +211,9 @@ def test_msi_provider(): @staticmethod def test_user_pass_provider(): - username = env_utils.get_env("USER_NAME") - password = env_utils.get_env("USER_PASS") - auth = env_utils.get_env("USER_AUTH_ID", default="organizations") + username = get_env("USER_NAME") + password = get_env("USER_PASS") + auth = get_env("USER_AUTH_ID", default="organizations") if username and password and auth: with UserPassTokenProvider(KUSTO_URI, auth, username, password) as provider: @@ -250,7 +250,7 @@ def test_interactive_login(): print(" *** Skipped interactive login Test ***") return - auth_id = env_utils.get_auth_id() + auth_id = get_auth_id() with InteractiveLoginTokenProvider(KUSTO_URI, auth_id) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -263,9 +263,9 @@ def test_interactive_login(): def test_app_key_provider(): # default details are for kusto-client-e2e-test-app # to run the test, get the key from Azure portal - app_id = env_utils.get_app_id() - auth_id = env_utils.get_auth_id() - app_key = env_utils.get_app_key() + app_id = get_app_id() + auth_id = get_auth_id() + app_key = get_app_key() if app_id and app_key and auth_id: with ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key) as provider: @@ -280,11 +280,11 @@ def test_app_key_provider(): @staticmethod def test_app_cert_provider(): - cert_app_id = env_utils.get_app_id(optional=True) - cert_auth = env_utils.get_auth_id(optional=True) - thumbprint = env_utils.get_env("CERT_THUMBPRINT", optional=True) - public_cert_path = env_utils.get_env("CERT_PUBLIC_CERT_PATH", optional=True) - pem_key_path = env_utils.get_env("CERT_PEM_KEY_PATH", optional=True) + cert_app_id = get_app_id(optional=True) + cert_auth = get_auth_id(optional=True) + thumbprint = get_env("CERT_THUMBPRINT", optional=True) + public_cert_path = get_env("CERT_PUBLIC_CERT_PATH", optional=True) + pem_key_path = get_env("CERT_PEM_KEY_PATH", optional=True) if pem_key_path and thumbprint and cert_app_id: with open(pem_key_path, "rb") as file: @@ -359,9 +359,9 @@ def test_cloud_mfa_on(): @staticmethod def test_azure_identity_default_token_provider(): - app_id = env_utils.get_app_id() - auth_id = env_utils.get_auth_id() - app_key = env_utils.get_app_key() + app_id = get_app_id() + auth_id = get_auth_id() + app_key = get_app_key() with AzureIdentityTokenCredentialProvider(KUSTO_URI, credential=DefaultAzureCredential()) as provider: token = provider.get_token() diff --git a/azure-kusto-ingest/tests/e2e.py b/azure-kusto-ingest/tests/e2e.py index 1cf083c3..d96a9b27 100644 --- a/azure-kusto-ingest/tests/e2e.py +++ b/azure-kusto-ingest/tests/e2e.py @@ -13,7 +13,7 @@ from azure.identity import DefaultAzureCredential from typing import Optional, ClassVar -import env_utils +from azure.kusto.data.env_utils import get_env, get_app_id, get_auth_id, get_app_key from azure.kusto.data import KustoClient, KustoConnectionStringBuilder from azure.kusto.data._token_providers import AsyncDefaultAzureCredential from azure.kusto.data.aio import KustoClient as AsyncKustoClient @@ -169,15 +169,15 @@ def dm_kcsb_from_env(cls) -> KustoConnectionStringBuilder: @classmethod def setup_class(cls): # DM CS can be composed from engine CS - cls.engine_cs = env_utils.get_env("ENGINE_CONNECTION_STRING") - cls.ai_engine_cs = env_utils.get_env("APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING") + cls.engine_cs = get_env("ENGINE_CONNECTION_STRING") + cls.ai_engine_cs = get_env("APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING") - cls.app_id = env_utils.get_app_id() - cls.auth_id = env_utils.get_auth_id() - cls.app_key = env_utils.get_app_key() + cls.app_id = get_app_id() + cls.auth_id = get_auth_id() + cls.app_key = get_app_key() - cls.test_db = env_utils.get_env("TEST_DATABASE") - cls.test_blob = env_utils.get_env("TEST_BLOB", optional=True) + cls.test_db = get_env("TEST_DATABASE") + cls.test_blob = get_env("TEST_BLOB", optional=True) # Init clients python_version = "_".join([str(v) for v in sys.version_info[:3]]) From eb7459e52756b57356f933c8d3e7692874ab31c7 Mon Sep 17 00:00:00 2001 From: AsafMah Date: Tue, 25 Jul 2023 11:22:48 +0300 Subject: [PATCH 03/13] Test --- azure-kusto-data/azure/kusto/data/_token_providers.py | 1 + azure-kusto-data/tests/aio/test_async_token_providers.py | 4 ++-- azure-kusto-data/tests/test_token_providers.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/azure-kusto-data/azure/kusto/data/_token_providers.py b/azure-kusto-data/azure/kusto/data/_token_providers.py index 687dca47..3ff77bda 100644 --- a/azure-kusto-data/azure/kusto/data/_token_providers.py +++ b/azure-kusto-data/azure/kusto/data/_token_providers.py @@ -234,6 +234,7 @@ def _get_token_from_cache_impl(self) -> Optional[dict]: @staticmethod def _valid_token_or_none(token: dict) -> Optional[dict]: if token is None or TokenConstants.MSAL_ERROR in token: + print("token error - " + str(token)) return None return token diff --git a/azure-kusto-data/tests/aio/test_async_token_providers.py b/azure-kusto-data/tests/aio/test_async_token_providers.py index b98610f9..c9c7ebcb 100644 --- a/azure-kusto-data/tests/aio/test_async_token_providers.py +++ b/azure-kusto-data/tests/aio/test_async_token_providers.py @@ -188,8 +188,8 @@ async def test_msi_provider(self): @aio_documented_by(TokenProviderTests.test_user_pass_provider) @pytest.mark.asyncio async def test_user_pass_provider(self): - username = get_env("USER_NAME") - password = get_env("USER_PASS") + username = get_env("USER_NAME", optional=True) + password = get_env("USER_PASS", optional=True) auth = get_env("USER_AUTH_ID", default="organizations") if username and password and auth: diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index 33009179..4d174c28 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -6,8 +6,8 @@ from asgiref.sync import async_to_sync from azure.identity import ClientSecretCredential, DefaultAzureCredential -from azure.kusto.data.env_utils import get_env, get_app_id, get_auth_id, get_app_key from azure.kusto.data._token_providers import * +from azure.kusto.data.env_utils import get_env, get_app_id, get_auth_id, get_app_key KUSTO_URI = "https://sdkse2etest.eastus.kusto.windows.net" TOKEN_VALUE = "little miss sunshine" @@ -211,8 +211,8 @@ def test_msi_provider(): @staticmethod def test_user_pass_provider(): - username = get_env("USER_NAME") - password = get_env("USER_PASS") + username = get_env("USER_NAME", optional=True) + password = get_env("USER_PASS", optional=True) auth = get_env("USER_AUTH_ID", default="organizations") if username and password and auth: From 493f585cd317651142148864e33c6d53178aa932 Mon Sep 17 00:00:00 2001 From: AsafMah Date: Tue, 25 Jul 2023 11:26:47 +0300 Subject: [PATCH 04/13] Test --- azure-kusto-data/azure/kusto/data/_token_providers.py | 1 - azure-kusto-data/tests/test_token_providers.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-kusto-data/azure/kusto/data/_token_providers.py b/azure-kusto-data/azure/kusto/data/_token_providers.py index 3ff77bda..687dca47 100644 --- a/azure-kusto-data/azure/kusto/data/_token_providers.py +++ b/azure-kusto-data/azure/kusto/data/_token_providers.py @@ -234,7 +234,6 @@ def _get_token_from_cache_impl(self) -> Optional[dict]: @staticmethod def _valid_token_or_none(token: dict) -> Optional[dict]: if token is None or TokenConstants.MSAL_ERROR in token: - print("token error - " + str(token)) return None return token diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index 4d174c28..9e9f8337 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -274,6 +274,7 @@ def test_app_key_provider(): # Again through cache token = provider._get_token_from_cache_impl() + print("token error - " + str(token)) assert TokenProviderTests.get_token_value(token) is not None else: print(" *** Skipped App Id & Key Provider Test ***") From f7f627377299874c578b44229aa1e99f33706db1 Mon Sep 17 00:00:00 2001 From: AsafMah Date: Tue, 25 Jul 2023 11:34:33 +0300 Subject: [PATCH 05/13] Remove account=None calls --- azure-kusto-data/azure/kusto/data/_token_providers.py | 4 ++-- azure-kusto-data/tests/test_token_providers.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/azure-kusto-data/azure/kusto/data/_token_providers.py b/azure-kusto-data/azure/kusto/data/_token_providers.py index 687dca47..36d95e19 100644 --- a/azure-kusto-data/azure/kusto/data/_token_providers.py +++ b/azure-kusto-data/azure/kusto/data/_token_providers.py @@ -565,7 +565,7 @@ def _get_token_impl(self) -> Optional[dict]: return self._valid_token_or_throw(token) def _get_token_from_cache_impl(self) -> dict: - token = self._msal_client.acquire_token_silent(scopes=self._scopes, account=None) + token = self._msal_client.acquire_token_for_client(scopes=self._scopes) return self._valid_token_or_none(token) @@ -614,7 +614,7 @@ def _get_token_impl(self) -> Optional[dict]: return self._valid_token_or_throw(token) def _get_token_from_cache_impl(self) -> dict: - token = self._msal_client.acquire_token_silent(scopes=self._scopes, account=None) + token = self._msal_client.acquire_token_for_client(scopes=self._scopes) return self._valid_token_or_none(token) diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index 9e9f8337..4d174c28 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -274,7 +274,6 @@ def test_app_key_provider(): # Again through cache token = provider._get_token_from_cache_impl() - print("token error - " + str(token)) assert TokenProviderTests.get_token_value(token) is not None else: print(" *** Skipped App Id & Key Provider Test ***") From 3f86f86de69c1a4cdf7efac00794c24b29f9bb07 Mon Sep 17 00:00:00 2001 From: AsafMah Date: Tue, 25 Jul 2023 11:37:01 +0300 Subject: [PATCH 06/13] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df73b5a1..2a3810d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +### Fixed +- Internal fixes for environment variables + ## [4.2.0] - 2023-04-16 ### Added - Added Initial Catalog (Default Database) parameter to ConnectionStringBuilder From 37b05c55dcd0c574c5fc20bf9d4b2dc18faa8622 Mon Sep 17 00:00:00 2001 From: AsafMah Date: Tue, 25 Jul 2023 11:38:26 +0300 Subject: [PATCH 07/13] Fixed e2e --- azure-kusto-ingest/tests/e2e.py | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-kusto-ingest/tests/e2e.py b/azure-kusto-ingest/tests/e2e.py index d96a9b27..c92c1e6d 100644 --- a/azure-kusto-ingest/tests/e2e.py +++ b/azure-kusto-ingest/tests/e2e.py @@ -170,6 +170,7 @@ def dm_kcsb_from_env(cls) -> KustoConnectionStringBuilder: def setup_class(cls): # DM CS can be composed from engine CS cls.engine_cs = get_env("ENGINE_CONNECTION_STRING") + cls.dm_cs = get_env("DM_CONNECTION_STRING", default=cls.engine_cs.replace("//", "//ingest-")) cls.ai_engine_cs = get_env("APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING") cls.app_id = get_app_id() From 7696957b347192cd4d9d5e0da5d7033375c56f32 Mon Sep 17 00:00:00 2001 From: AsafMah Date: Wed, 26 Jul 2023 01:27:11 +0300 Subject: [PATCH 08/13] Pr fixes --- azure-kusto-data/azure/kusto/data/_token_providers.py | 10 ++++------ .../tests/aio/test_async_token_providers.py | 6 +++--- azure-kusto-data/tests/test_token_providers.py | 6 +++--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/azure-kusto-data/azure/kusto/data/_token_providers.py b/azure-kusto-data/azure/kusto/data/_token_providers.py index 36d95e19..705bc789 100644 --- a/azure-kusto-data/azure/kusto/data/_token_providers.py +++ b/azure-kusto-data/azure/kusto/data/_token_providers.py @@ -564,9 +564,8 @@ def _get_token_impl(self) -> Optional[dict]: token = self._msal_client.acquire_token_for_client(scopes=self._scopes) return self._valid_token_or_throw(token) - def _get_token_from_cache_impl(self) -> dict: - token = self._msal_client.acquire_token_for_client(scopes=self._scopes) - return self._valid_token_or_none(token) + def _get_token_from_cache_impl(self) -> None: + return None class ApplicationCertificateTokenProvider(CloudInfoTokenProvider): @@ -613,9 +612,8 @@ def _get_token_impl(self) -> Optional[dict]: token = self._msal_client.acquire_token_for_client(scopes=self._scopes) return self._valid_token_or_throw(token) - def _get_token_from_cache_impl(self) -> dict: - token = self._msal_client.acquire_token_for_client(scopes=self._scopes) - return self._valid_token_or_none(token) + def _get_token_from_cache_impl(self) -> None: + return None class AzureIdentityTokenCredentialProvider(CloudInfoTokenProvider): diff --git a/azure-kusto-data/tests/aio/test_async_token_providers.py b/azure-kusto-data/tests/aio/test_async_token_providers.py index c9c7ebcb..5620cf83 100644 --- a/azure-kusto-data/tests/aio/test_async_token_providers.py +++ b/azure-kusto-data/tests/aio/test_async_token_providers.py @@ -161,8 +161,8 @@ async def test_msi_provider(self): print(" *** Skipped MSI Provider Test ***") return - user_msi_object_id = get_env("MSI_OBJECT_ID") - user_msi_client_id = get_env("MSI_CLIENT_ID") + user_msi_object_id = get_env("MSI_OBJECT_ID", optional=True) + user_msi_client_id = get_env("MSI_CLIENT_ID", optional=True) # system MSI with MsiTokenProvider(KUSTO_URI, is_async=True) as provider: @@ -253,7 +253,7 @@ async def test_app_cert_provider(self): public_cert_path = get_env("CERT_PUBLIC_CERT_PATH", optional=True) pem_key_path = get_env("CERT_PEM_KEY_PATH", optional=True) - if pem_key_path and thumbprint and cert_app_id: + if pem_key_path and thumbprint and cert_app_id and cert_auth: with open(pem_key_path, "rb") as file: pem_key = file.read() diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index 4d174c28..9d54e431 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -185,8 +185,8 @@ def test_msi_provider(): print(" *** Skipped MSI Provider Test ***") return - user_msi_object_id = get_env("MSI_OBJECT_ID") - user_msi_client_id = get_env("MSI_CLIENT_ID") + user_msi_object_id = get_env("MSI_OBJECT_ID", optional=True) + user_msi_client_id = get_env("MSI_CLIENT_ID", optional=True) # system MSI with MsiTokenProvider(KUSTO_URI) as provider: @@ -286,7 +286,7 @@ def test_app_cert_provider(): public_cert_path = get_env("CERT_PUBLIC_CERT_PATH", optional=True) pem_key_path = get_env("CERT_PEM_KEY_PATH", optional=True) - if pem_key_path and thumbprint and cert_app_id: + if pem_key_path and thumbprint and cert_app_id and cert_auth: with open(pem_key_path, "rb") as file: pem_key = file.read() From c5908c67db01d023e4b0e79ff25e11dc67f0ba5c Mon Sep 17 00:00:00 2001 From: AsafMah Date: Wed, 26 Jul 2023 01:33:50 +0300 Subject: [PATCH 09/13] Fixed tests --- .../tests/aio/test_async_token_providers.py | 8 -------- azure-kusto-data/tests/test_token_providers.py | 12 ------------ 2 files changed, 20 deletions(-) diff --git a/azure-kusto-data/tests/aio/test_async_token_providers.py b/azure-kusto-data/tests/aio/test_async_token_providers.py index 5620cf83..7349c4d6 100644 --- a/azure-kusto-data/tests/aio/test_async_token_providers.py +++ b/azure-kusto-data/tests/aio/test_async_token_providers.py @@ -235,10 +235,6 @@ async def test_app_key_provider(self): with ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key, is_async=True) as provider: token = await provider.get_token_async() assert self.get_token_value(token) is not None - - # Again through cache - token = provider._get_token_from_cache_impl() - assert self.get_token_value(token) is not None else: print(" *** Skipped App Id & Key Provider Test ***") @@ -261,10 +257,6 @@ async def test_app_cert_provider(self): token = await provider.get_token_async() assert self.get_token_value(token) is not None - # Again through cache - token = provider._get_token_from_cache_impl() - assert self.get_token_value(token) is not None - if public_cert_path: with open(public_cert_path, "r") as file: public_cert = file.read() diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index 9d54e431..651b0791 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -271,10 +271,6 @@ def test_app_key_provider(): with ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None - - # Again through cache - token = provider._get_token_from_cache_impl() - assert TokenProviderTests.get_token_value(token) is not None else: print(" *** Skipped App Id & Key Provider Test ***") @@ -294,10 +290,6 @@ def test_app_cert_provider(): token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None - # Again through cache - token = provider._get_token_from_cache_impl() - assert TokenProviderTests.get_token_value(token) is not None - if public_cert_path: with open(public_cert_path, "r") as file: public_cert = file.read() @@ -305,10 +297,6 @@ def test_app_cert_provider(): with ApplicationCertificateTokenProvider(KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint, public_cert) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None - - # Again through cache - token = provider._get_token_from_cache_impl() - assert TokenProviderTests.get_token_value(token) is not None else: print(" *** Skipped App Cert SNI Provider Test ***") From d44659bf1dcbe441735fb97ad4ea79e4b8b0b517 Mon Sep 17 00:00:00 2001 From: AsafMah Date: Sun, 30 Jul 2023 08:34:17 +0300 Subject: [PATCH 10/13] Make engine connection string configurable --- azure-kusto-data/tests/test_token_providers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index 651b0791..b8a1dcdd 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -9,7 +9,6 @@ from azure.kusto.data._token_providers import * from azure.kusto.data.env_utils import get_env, get_app_id, get_auth_id, get_app_key -KUSTO_URI = "https://sdkse2etest.eastus.kusto.windows.net" TOKEN_VALUE = "little miss sunshine" TEST_AZ_AUTH = False # enable this in environments with az cli installed, and make sure to call 'az login' first @@ -46,6 +45,8 @@ def _get_token_from_cache_impl(self) -> Optional[dict]: class TokenProviderTests(unittest.TestCase): + KUSTO_URI = get_env("ENGINE_CONNECTION_STRING") + @staticmethod def test_base_provider(): # test init with no URI From cebeeda9d7dde1ac7755b1bbffae7d66438e39bd Mon Sep 17 00:00:00 2001 From: AsafMah Date: Sun, 30 Jul 2023 08:38:04 +0300 Subject: [PATCH 11/13] Fix build file --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4542d556..d5813696 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,6 +41,7 @@ jobs: APP_ID: ${{ secrets.APP_ID }} APP_KEY: ${{ secrets.APP_KEY }} AUTH_ID: ${{ secrets.AUTH_ID }} + ENGINE_CONNECTION_STRING: ${{ secrets.ENGINE_CONNECTION_STRING }} - name: EtoE Test with pytest env: APP_ID: ${{ secrets.APP_ID }} From b5ab85eefb8368ccd9947e60143476f5026423d4 Mon Sep 17 00:00:00 2001 From: AsafMah Date: Sun, 30 Jul 2023 08:39:59 +0300 Subject: [PATCH 12/13] Fix build file --- .../tests/test_token_providers.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index b8a1dcdd..6d403e8b 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -45,7 +45,7 @@ def _get_token_from_cache_impl(self) -> Optional[dict]: class TokenProviderTests(unittest.TestCase): - KUSTO_URI = get_env("ENGINE_CONNECTION_STRING") + KUSTO_URI = get_env("ENGINE_CONNECTION_STRING", optional=True) @staticmethod def test_base_provider(): @@ -172,7 +172,7 @@ def test_az_provider(): return print("Note!\nThe test 'test_az_provider' will fail if 'az login' was not called.") - with AzCliTokenProvider(KUSTO_URI) as provider: + with AzCliTokenProvider(self.KUSTO_URI) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -190,13 +190,13 @@ def test_msi_provider(): user_msi_client_id = get_env("MSI_CLIENT_ID", optional=True) # system MSI - with MsiTokenProvider(KUSTO_URI) as provider: + with MsiTokenProvider(self.KUSTO_URI) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None if user_msi_object_id is not None: args = {"object_id": user_msi_object_id} - with MsiTokenProvider(KUSTO_URI, args) as provider: + with MsiTokenProvider(self.KUSTO_URI, args) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None else: @@ -204,7 +204,7 @@ def test_msi_provider(): if user_msi_client_id is not None: args = {"client_id": user_msi_client_id} - with MsiTokenProvider(KUSTO_URI, args) as provider: + with MsiTokenProvider(self.KUSTO_URI, args) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None else: @@ -217,7 +217,7 @@ def test_user_pass_provider(): auth = get_env("USER_AUTH_ID", default="organizations") if username and password and auth: - with UserPassTokenProvider(KUSTO_URI, auth, username, password) as provider: + with UserPassTokenProvider(self.KUSTO_URI, auth, username, password) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -237,7 +237,7 @@ def callback(x, x2, x3): # break here if you debug this test, and get the code from 'x' print(f"Please go to {x} and enter code {x2} to authenticate, expires in {x3}") - with DeviceLoginTokenProvider(KUSTO_URI, "organizations", callback) as provider: + with DeviceLoginTokenProvider(self.KUSTO_URI, "organizations", callback) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -252,7 +252,7 @@ def test_interactive_login(): return auth_id = get_auth_id() - with InteractiveLoginTokenProvider(KUSTO_URI, auth_id) as provider: + with InteractiveLoginTokenProvider(self.KUSTO_URI, auth_id) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -269,7 +269,7 @@ def test_app_key_provider(): app_key = get_app_key() if app_id and app_key and auth_id: - with ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key) as provider: + with ApplicationKeyTokenProvider(self.KUSTO_URI, auth_id, app_id, app_key) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None else: @@ -287,7 +287,7 @@ def test_app_cert_provider(): with open(pem_key_path, "rb") as file: pem_key = file.read() - with ApplicationCertificateTokenProvider(KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint) as provider: + with ApplicationCertificateTokenProvider(self.KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -295,7 +295,7 @@ def test_app_cert_provider(): with open(public_cert_path, "r") as file: public_cert = file.read() - with ApplicationCertificateTokenProvider(KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint, public_cert) as provider: + with ApplicationCertificateTokenProvider(self.KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint, public_cert) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None else: @@ -352,12 +352,12 @@ def test_azure_identity_default_token_provider(): auth_id = get_auth_id() app_key = get_app_key() - with AzureIdentityTokenCredentialProvider(KUSTO_URI, credential=DefaultAzureCredential()) as provider: + with AzureIdentityTokenCredentialProvider(self.KUSTO_URI, credential=DefaultAzureCredential()) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None with AzureIdentityTokenCredentialProvider( - KUSTO_URI, + self.KUSTO_URI, credential_from_login_endpoint=lambda login_endpoint: ClientSecretCredential( authority=login_endpoint, client_id=app_id, client_secret=app_key, tenant_id=auth_id ), From 1f2b00dff3f60237f8e32355757842a53757acdc Mon Sep 17 00:00:00 2001 From: AsafMah Date: Sun, 30 Jul 2023 08:45:48 +0300 Subject: [PATCH 13/13] Fix again --- .../tests/test_token_providers.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/azure-kusto-data/tests/test_token_providers.py b/azure-kusto-data/tests/test_token_providers.py index 6d403e8b..3fe37a7c 100644 --- a/azure-kusto-data/tests/test_token_providers.py +++ b/azure-kusto-data/tests/test_token_providers.py @@ -44,9 +44,10 @@ def _get_token_from_cache_impl(self) -> Optional[dict]: return None -class TokenProviderTests(unittest.TestCase): - KUSTO_URI = get_env("ENGINE_CONNECTION_STRING", optional=True) +KUSTO_URI = get_env("ENGINE_CONNECTION_STRING", None) + +class TokenProviderTests(unittest.TestCase): @staticmethod def test_base_provider(): # test init with no URI @@ -172,7 +173,7 @@ def test_az_provider(): return print("Note!\nThe test 'test_az_provider' will fail if 'az login' was not called.") - with AzCliTokenProvider(self.KUSTO_URI) as provider: + with AzCliTokenProvider(KUSTO_URI) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -190,13 +191,13 @@ def test_msi_provider(): user_msi_client_id = get_env("MSI_CLIENT_ID", optional=True) # system MSI - with MsiTokenProvider(self.KUSTO_URI) as provider: + with MsiTokenProvider(KUSTO_URI) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None if user_msi_object_id is not None: args = {"object_id": user_msi_object_id} - with MsiTokenProvider(self.KUSTO_URI, args) as provider: + with MsiTokenProvider(KUSTO_URI, args) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None else: @@ -204,7 +205,7 @@ def test_msi_provider(): if user_msi_client_id is not None: args = {"client_id": user_msi_client_id} - with MsiTokenProvider(self.KUSTO_URI, args) as provider: + with MsiTokenProvider(KUSTO_URI, args) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None else: @@ -217,7 +218,7 @@ def test_user_pass_provider(): auth = get_env("USER_AUTH_ID", default="organizations") if username and password and auth: - with UserPassTokenProvider(self.KUSTO_URI, auth, username, password) as provider: + with UserPassTokenProvider(KUSTO_URI, auth, username, password) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -237,7 +238,7 @@ def callback(x, x2, x3): # break here if you debug this test, and get the code from 'x' print(f"Please go to {x} and enter code {x2} to authenticate, expires in {x3}") - with DeviceLoginTokenProvider(self.KUSTO_URI, "organizations", callback) as provider: + with DeviceLoginTokenProvider(KUSTO_URI, "organizations", callback) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -252,7 +253,7 @@ def test_interactive_login(): return auth_id = get_auth_id() - with InteractiveLoginTokenProvider(self.KUSTO_URI, auth_id) as provider: + with InteractiveLoginTokenProvider(KUSTO_URI, auth_id) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -269,7 +270,7 @@ def test_app_key_provider(): app_key = get_app_key() if app_id and app_key and auth_id: - with ApplicationKeyTokenProvider(self.KUSTO_URI, auth_id, app_id, app_key) as provider: + with ApplicationKeyTokenProvider(KUSTO_URI, auth_id, app_id, app_key) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None else: @@ -287,7 +288,7 @@ def test_app_cert_provider(): with open(pem_key_path, "rb") as file: pem_key = file.read() - with ApplicationCertificateTokenProvider(self.KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint) as provider: + with ApplicationCertificateTokenProvider(KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None @@ -295,7 +296,7 @@ def test_app_cert_provider(): with open(public_cert_path, "r") as file: public_cert = file.read() - with ApplicationCertificateTokenProvider(self.KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint, public_cert) as provider: + with ApplicationCertificateTokenProvider(KUSTO_URI, cert_app_id, cert_auth, pem_key, thumbprint, public_cert) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None else: @@ -352,12 +353,12 @@ def test_azure_identity_default_token_provider(): auth_id = get_auth_id() app_key = get_app_key() - with AzureIdentityTokenCredentialProvider(self.KUSTO_URI, credential=DefaultAzureCredential()) as provider: + with AzureIdentityTokenCredentialProvider(KUSTO_URI, credential=DefaultAzureCredential()) as provider: token = provider.get_token() assert TokenProviderTests.get_token_value(token) is not None with AzureIdentityTokenCredentialProvider( - self.KUSTO_URI, + KUSTO_URI, credential_from_login_endpoint=lambda login_endpoint: ClientSecretCredential( authority=login_endpoint, client_id=app_id, client_secret=app_key, tenant_id=auth_id ),