From 5b2aa0da89c1e65c815e6d30a60e38074cc37f9a Mon Sep 17 00:00:00 2001 From: Michael Matzka <14311597+mima0815@users.noreply.github.com> Date: Thu, 28 Mar 2024 13:04:11 +0100 Subject: [PATCH 1/2] feat(request): add request retries in case of server errors --- .github/workflows/ci.yml | 11 ++- README.md | 92 +++++++++++++------ .../sdk/utils/authentication/confidential.py | 22 ++++- .../utils/authentication/test_confidential.py | 16 ++-- 4 files changed, 97 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9291646..ec9487a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: pull_request: - branches: [main] + branches: [ main ] push: - branches: [main] + branches: [ main ] jobs: build: @@ -18,7 +18,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.11" - name: Install Poetry uses: snok/install-poetry@v1 @@ -43,9 +43,10 @@ jobs: name: Test runs-on: ubuntu-20.04 strategy: + fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - poetry-version: ["1.6.1"] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + poetry-version: [ "1.8.2" ] include: - python-version: "3.7" poetry-version: "1.5.1" diff --git a/README.md b/README.md index de969c9..de0ad7f 100644 --- a/README.md +++ b/README.md @@ -5,19 +5,20 @@ [![PyPi](https://img.shields.io/pypi/v/fds.sdk.utils)](https://pypi.org/project/fds.sdk.utils/) [![Apache-2 license](https://img.shields.io/badge/license-Apache2-brightgreen.svg)](https://www.apache.org/licenses/LICENSE-2.0) -This repository contains a collection of utilities that supports FactSet's SDK in Python and facilitate usage of FactSet APIs. +This repository contains a collection of utilities that supports FactSet's SDK in Python and facilitate usage of FactSet +APIs. ## Installation ### Poetry -```python +```sh poetry add fds.sdk.utils ``` ### pip -```python +```sh pip install fds.sdk.utils ``` @@ -29,7 +30,8 @@ This library contains multiple modules, sample usage of each module is below. First, you need to create the OAuth 2.0 client configuration that will be used to authenticate against FactSet's APIs: -1. [Create a new application](https://developer.factset.com/learn/authentication-oauth2#creating-an-application) on FactSet's Developer Portal. +1. [Create a new application](https://developer.factset.com/learn/authentication-oauth2#creating-an-application) on + FactSet's Developer Portal. 2. When prompted, download the configuration file and move it to your development environment. ```python @@ -61,7 +63,7 @@ from fds.sdk.utils.authentication import ConfidentialClient client = ConfidentialClient( config_path='/path/to/config.json', - proxy= "http://secret:password@localhost:5050", + proxy="http://secret:password@localhost:5050", proxy_headers={ "Custom-Proxy-Header": "Custom-Proxy-Header-Value" } @@ -79,7 +81,6 @@ With `verify_ssl` it is possible to disable the verifications of certificates. Disabling the verification is not recommended, but it might be useful during local development or testing - ```python from fds.sdk.utils.authentication import ConfidentialClient @@ -90,55 +91,86 @@ client = ConfidentialClient( ) ``` +### Request Retries + +In case the request retry behaviour should be customized, it is possible to pass a `urllib3.Retry` object to +the `ConfidentialClient`. + +```python +from urllib3 import Retry +from fds.sdk.utils.authentication import ConfidentialClient + +client = ConfidentialClient( + config_path='/path/to/config.json', + retry=Retry( + total=5, + backoff_factor=0.1, + status_forcelist=[500, 502, 503, 504] + ) +) +``` + ## Modules Information about the various utility modules contained in this library can be found below. ### Authentication -The [authentication module](src/fds/sdk/utils/authentication) provides helper classes that facilitate [OAuth 2.0](https://developer.factset.com/learn/authentication-oauth2) authentication and authorization with FactSet's APIs. Currently the module has support for the [client credentials flow](https://github.com/factset/oauth2-guidelines#client-credentials-flow-1). +The [authentication module](src/fds/sdk/utils/authentication) provides helper classes that +facilitate [OAuth 2.0](https://developer.factset.com/learn/authentication-oauth2) authentication and authorization with +FactSet's APIs. Currently the module has support for +the [client credentials flow](https://github.com/factset/oauth2-guidelines#client-credentials-flow-1). Each helper class in the module has the following features: -* Accepts a configuration file or `dict` that contains information about the OAuth 2.0 client, including the client ID and private key. +* Accepts a configuration file or `dict` that contains information about the OAuth 2.0 client, including the client ID + and private key. * Performs authentication with FactSet's OAuth 2.0 authorization server and retrieves an access token. * Caches the access token for reuse and requests a new access token as needed when one expires. * In order for this to work correctly, the helper class instance should be reused in production environments. #### Configuration -Classes in the authentication module require OAuth 2.0 client configuration information to be passed to constructors through a JSON-formatted file or a `dict`. In either case the format is the same: +Classes in the authentication module require OAuth 2.0 client configuration information to be passed to constructors +through a JSON-formatted file or a `dict`. In either case the format is the same: ```json { - "name": "Application name registered with FactSet's Developer Portal", - "clientId": "OAuth 2.0 Client ID registered with FactSet's Developer Portal", - "clientAuthType": "Confidential", - "owners": ["USERNAME-SERIAL"], - "jwk": { - "kty": "RSA", - "use": "sig", - "alg": "RS256", - "kid": "Key ID", - "d": "ECC Private Key", - "n": "Modulus", - "e": "Exponent", - "p": "First Prime Factor", - "q": "Second Prime Factor", - "dp": "First Factor CRT Exponent", - "dq": "Second Factor CRT Exponent", - "qi": "First CRT Coefficient", - } + "name": "Application name registered with FactSet's Developer Portal", + "clientId": "OAuth 2.0 Client ID registered with FactSet's Developer Portal", + "clientAuthType": "Confidential", + "owners": [ + "USERNAME-SERIAL" + ], + "jwk": { + "kty": "RSA", + "use": "sig", + "alg": "RS256", + "kid": "Key ID", + "d": "ECC Private Key", + "n": "Modulus", + "e": "Exponent", + "p": "First Prime Factor", + "q": "Second Prime Factor", + "dp": "First Factor CRT Exponent", + "dq": "Second Factor CRT Exponent", + "qi": "First CRT Coefficient" + } } ``` -If you're just starting out, you can visit FactSet's Developer Portal to [create a new application](https://developer.factset.com/applications) and download a configuration file in this format. +If you're just starting out, you can visit FactSet's Developer Portal +to [create a new application](https://developer.factset.com/applications) and download a configuration file in this +format. -If you're creating and managing your signing key pair yourself, see the required [JWK parameters](https://github.com/factset/oauth2-guidelines#jwk-parameters) for public-private key pairs. +If you're creating and managing your signing key pair yourself, see the +required [JWK parameters](https://github.com/factset/oauth2-guidelines#jwk-parameters) for public-private key pairs. ## Debugging -This library uses the [logging module](https://docs.python.org/3/howto/logging.html) to log various messages that will help you understand what it's doing. You can increase the log level to see additional debug information using standard conventions. For example: +This library uses the [logging module](https://docs.python.org/3/howto/logging.html) to log various messages that will +help you understand what it's doing. You can increase the log level to see additional debug information using standard +conventions. For example: ```python logging.getLogger('fds.sdk.utils').setLevel(logging.DEBUG) diff --git a/src/fds/sdk/utils/authentication/confidential.py b/src/fds/sdk/utils/authentication/confidential.py index a38b7c5..39f04cb 100644 --- a/src/fds/sdk/utils/authentication/confidential.py +++ b/src/fds/sdk/utils/authentication/confidential.py @@ -6,7 +6,10 @@ import requests from jose import JWSError, jws from oauthlib.oauth2 import BackendApplicationClient +from requests import Session +from requests.adapters import HTTPAdapter from requests_oauthlib import OAuth2Session +from urllib3 import Retry from .constants import CONSTS from .oauth2client import OAuth2Client @@ -41,6 +44,7 @@ def __init__( proxy_headers: dict = None, verify_ssl: bool = True, ssl_ca_cert: str = None, + retry: Retry = None, ) -> None: """ Creates a new ConfidentialClient. @@ -95,6 +99,8 @@ def __init__( `ssl_ca_cert` (str): Set this to customize the certificate file to verify the peer. If ``ssl_ca_cert`` is set, the ca_cert will be verified whether ``verify_ssl`` is enabled + `retry` (Retry): Set this to custommize the retry policy for the requests. If not set, the default is used. + Raises: AuthServerMetadataError: Raised if there's an issue retrieving the authorization server metadata @@ -131,10 +137,21 @@ def __init__( self._proxy_headers = proxy_headers self._ssl_ca_cert = ssl_ca_cert + if retry is not None: + self._retry = retry + else: + self._retry = Retry( + total=3, + backoff_factor=0.1, + status_forcelist=[413, 429, 500, 502, 503, 504], + allowed_methods={"DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT", "TRACE"}, + ) + try: self._oauth_session = OAuth2Session( client=BackendApplicationClient(client_id=self._config[CONSTS.CONFIG_CLIENT_ID]) ) + self._oauth_session.mount("https://", HTTPAdapter(max_retries=self._retry)) except Exception as e: raise ConfidentialClientError( f"Error instantiating OAuth2 session with {CONSTS.CONFIG_CLIENT_ID}:{self._config[CONSTS.CONFIG_CLIENT_ID]}" @@ -152,6 +169,9 @@ def __init__( log.debug("Credentials are complete and formatted correctly") + self._requests_session = Session() + self._requests_session.mount("https://", HTTPAdapter(max_retries=self._retry)) + self._init_auth_server_metadata() self._cached_token = {} @@ -167,7 +187,7 @@ def _init_auth_server_metadata(self) -> None: if self._ssl_ca_cert: verify = self._ssl_ca_cert - res = requests.get( + res = self._requests_session.get( url=self._config[CONSTS.CONFIG_WELL_KNOWN_URI], proxies=self._proxy, verify=verify, diff --git a/tests/fds/sdk/utils/authentication/test_confidential.py b/tests/fds/sdk/utils/authentication/test_confidential.py index 821cdea..b43b0df 100644 --- a/tests/fds/sdk/utils/authentication/test_confidential.py +++ b/tests/fds/sdk/utils/authentication/test_confidential.py @@ -46,7 +46,7 @@ def client(mocker, example_config): "fds.sdk.utils.authentication.confidential.OAuth2Session.fetch_token", return_value={"access_token": "test-token", "expires_at": 10}, ) - mock_get = mocker.patch("fds.sdk.utils.authentication.confidential.requests.get") + mock_get = mocker.patch("fds.sdk.utils.authentication.confidential.requests.Session.get") mock_get.return_value.json.return_value = { "issuer": "test-issuer", "token_endpoint": "https://test.token.endpoint", @@ -75,7 +75,7 @@ def json(self): return {"issuer": "test", "token_endpoint": "http://test.test"} caplog.set_level(logging.DEBUG) - mocker.patch("requests.get", return_value=AuthServerMetadataRes()) + mocker.patch("requests.Session.get", return_value=AuthServerMetadataRes()) client = ConfidentialClient(config=example_config) @@ -104,7 +104,7 @@ def json(self): return {"issuer": "test", "token_endpoint": "http://test.test"} caplog.set_level(logging.DEBUG) - mocker.patch("requests.get", return_value=AuthServerMetadataRes()) + mocker.patch("requests.Session.get", return_value=AuthServerMetadataRes()) mocker.patch("json.load", return_value=example_config) fake_file_path = "/my/fake/path/creds.json" @@ -189,7 +189,7 @@ class AuthServerMetadataRes: def json(self): return {"issuer": "test", "token_endpoint": "http://test.test"} - get_mock = mocker.patch("requests.get", return_value=AuthServerMetadataRes()) + get_mock = mocker.patch("requests.Session.get", return_value=AuthServerMetadataRes()) ConfidentialClient(config=example_config, **additional_parameters) @@ -218,7 +218,7 @@ class AuthServerMetadataRes: def json(self): return {"issuer": "test", "token_endpoint": "http://test.test"} - get_mock = mocker.patch("requests.get", return_value=AuthServerMetadataRes()) + get_mock = mocker.patch("requests.Session.get", return_value=AuthServerMetadataRes()) auth_test = "https://auth.test" example_config["wellKnownUri"] = auth_test @@ -237,7 +237,7 @@ def test_constructor_metadata_error(mocker, example_config): "fds.sdk.utils.authentication.confidential.OAuth2Session.fetch_token", return_value={"access_token": "test", "expires_at": 10}, ) - mocker.patch("requests.get", side_effect=Exception("error")) + mocker.patch("requests.Session.get", side_effect=Exception("error")) with pytest.raises(AuthServerMetadataError): ConfidentialClient(config=example_config) @@ -256,7 +256,7 @@ class AuthServerMetadataRes: def json(): return {} - mocker.patch("requests.get", return_value=AuthServerMetadataRes) + mocker.patch("requests.Session.get", return_value=AuthServerMetadataRes) with pytest.raises(AuthServerMetadataContentError): ConfidentialClient(config=example_config) @@ -392,7 +392,7 @@ def test_get_access_token_fetch_error(client, mocker, caplog): def test_get_access_token_cached(example_config, mocker, caplog): caplog.set_level(logging.DEBUG) - mock_get = mocker.patch("fds.sdk.utils.authentication.confidential.requests.get") + mock_get = mocker.patch("fds.sdk.utils.authentication.confidential.requests.Session.get") mock_get.return_value.json.return_value = { "issuer": "test-issuer", "token_endpoint": "https://test.token.endpoint", From 81327b92265da9ff309f5e9bd608bd7877510828 Mon Sep 17 00:00:00 2001 From: Michael Matzka <14311597+mima0815@users.noreply.github.com> Date: Fri, 5 Apr 2024 11:12:36 +0200 Subject: [PATCH 2/2] chore: change backoff factor --- src/fds/sdk/utils/authentication/confidential.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fds/sdk/utils/authentication/confidential.py b/src/fds/sdk/utils/authentication/confidential.py index 39f04cb..758ccc3 100644 --- a/src/fds/sdk/utils/authentication/confidential.py +++ b/src/fds/sdk/utils/authentication/confidential.py @@ -142,7 +142,7 @@ def __init__( else: self._retry = Retry( total=3, - backoff_factor=0.1, + backoff_factor=1, status_forcelist=[413, 429, 500, 502, 503, 504], allowed_methods={"DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT", "TRACE"}, )