From 93dfa9cb99225d6765b64636b883b1bd06a293b7 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Mon, 22 Mar 2021 22:18:33 -0400 Subject: [PATCH 1/2] work around AAD service principal race condition --- src/deployment/deploy.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index ee9a88e467..47927d61b3 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -300,7 +300,22 @@ def setup_rbac(self) -> None: service_principal_type="Application", app_id=app.app_id, ) - client.service_principals.create(service_principal_params) + try: + client.service_principals.create(service_principal_params) + except GraphErrorException as err: + # work around timing issue when creating service principal + # https://github.com/Azure/azure-cli/issues/14767 + if ( + "service principal being created must in the local tenant" + not in str(err) + ): + raise err + logging.warning( + "creating service principal failed with an error that occurs " + "due to AAD race conditions. retrying after delay" + ) + time.sleep(60) + client.service_principals.create(service_principal_params) else: app = existing[0] existing_role_values = [app_role.value for app_role in app.app_roles] From ddc561a5045249dcacc032a04e6629cfd8fa4978 Mon Sep 17 00:00:00 2001 From: Brian Caswell Date: Tue, 23 Mar 2021 16:04:23 -0400 Subject: [PATCH 2/2] try retrying more frequently --- src/deployment/deploy.py | 43 +++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/deployment/deploy.py b/src/deployment/deploy.py index 47927d61b3..5a47fa3d54 100644 --- a/src/deployment/deploy.py +++ b/src/deployment/deploy.py @@ -300,22 +300,33 @@ def setup_rbac(self) -> None: service_principal_type="Application", app_id=app.app_id, ) - try: - client.service_principals.create(service_principal_params) - except GraphErrorException as err: - # work around timing issue when creating service principal - # https://github.com/Azure/azure-cli/issues/14767 - if ( - "service principal being created must in the local tenant" - not in str(err) - ): - raise err - logging.warning( - "creating service principal failed with an error that occurs " - "due to AAD race conditions. retrying after delay" - ) - time.sleep(60) - client.service_principals.create(service_principal_params) + + def try_sp_create() -> None: + error: Optional[Exception] = None + for _ in range(10): + try: + client.service_principals.create(service_principal_params) + return + except GraphErrorException as err: + # work around timing issue when creating service principal + # https://github.com/Azure/azure-cli/issues/14767 + if ( + "service principal being created must in the local tenant" + not in str(err) + ): + raise err + logging.warning( + "creating service principal failed with an error that occurs " + "due to AAD race conditions" + ) + time.sleep(60) + if error is None: + raise Exception("service principal creation failed") + else: + raise error + + try_sp_create() + else: app = existing[0] existing_role_values = [app_role.value for app_role in app.app_roles]