diff --git a/optimizely/config_manager.py b/optimizely/config_manager.py new file mode 100644 index 000000000..1eacd279a --- /dev/null +++ b/optimizely/config_manager.py @@ -0,0 +1,239 @@ +# Copyright 2019, Optimizely +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import requests +import threading +import time +from requests import codes as http_status_codes +from requests import exceptions as requests_exceptions + +from . import exceptions as optimizely_exceptions +from . import logger as optimizely_logger +from . import project_config +from .error_handler import NoOpErrorHandler as noop_error_handler +from .helpers import enums + + +ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) + + +class BaseConfigManager(ABC): + """ Base class for Optimizely's config manager. """ + + def __init__(self, + logger=None, + error_handler=None): + """ Initialize config manager. + + Args: + logger: Provides a logger instance. + error_handler: Provides a handle_error method to handle exceptions. + """ + self.logger = logger or optimizely_logger.adapt_logger(logger or optimizely_logger.NoOpLogger()) + self.error_handler = error_handler or noop_error_handler + + @abc.abstractmethod + def get_config(self): + """ Get config for use by optimizely.Optimizely. + The config should be an instance of project_config.ProjectConfig.""" + pass + + +class StaticConfigManager(BaseConfigManager): + """ Config manager that returns ProjectConfig based on provided datafile. """ + + def __init__(self, + datafile=None, + logger=None, + error_handler=None): + """ Initialize config manager. Datafile has to be provided to use. + + Args: + datafile: JSON string representing the Optimizely project. + logger: Provides a logger instance. + error_handler: Provides a handle_error method to handle exceptions. + """ + super(StaticConfigManager, self).__init__(logger=logger, error_handler=error_handler) + self._config = None + if datafile: + self._config = project_config.ProjectConfig(datafile, self.logger, self.error_handler) + + def get_config(self): + """ Returns instance of ProjectConfig. + + Returns: + ProjectConfig. + """ + return self._config + + +class PollingConfigManager(BaseConfigManager): + """ Config manager that polls for the datafile and updated ProjectConfig based on an update interval. """ + + def __init__(self, + sdk_key=None, + datafile=None, + update_interval=None, + url=None, + url_template=None, + logger=None, + error_handler=None): + """ Initialize config manager. One of sdk_key or url has to be set to be able to use. + + Args: + sdk_key: Optional string uniquely identifying the datafile. + datafile: Optional JSON string representing the project. + update_interval: Optional floating point number representing time interval in seconds + at which to request datafile and set ProjectConfig. + url: Optional string representing URL from where to fetch the datafile. If set it supersedes the sdk_key. + url_template: Optional string template which in conjunction with sdk_key + determines URL from where to fetch the datafile. + logger: Provides a logger instance. + error_handler: Provides a handle_error method to handle exceptions. + """ + super(PollingConfigManager, self).__init__(logger=logger, error_handler=error_handler) + self.datafile_url = self.get_datafile_url(sdk_key, url, + url_template or enums.ConfigManager.DATAFILE_URL_TEMPLATE) + self.set_update_interval(update_interval) + self.last_modified = None + self._datafile = datafile + self._config = None + self._polling_thread = threading.Thread(target=self._run) + self._polling_thread.setDaemon(True) + if self._datafile: + self.set_config(self._datafile) + + @staticmethod + def get_datafile_url(sdk_key, url, url_template): + """ Helper method to determine URL from where to fetch the datafile. + + Args: + sdk_key: Key uniquely identifying the datafile. + url: String representing URL from which to fetch the datafile. + url_template: String representing template which is filled in with + SDK key to determine URL from which to fetch the datafile. + + Returns: + String representing URL to fetch datafile from. + + Raises: + optimizely.exceptions.InvalidInputException if: + - One of sdk_key or url is not provided. + - url_template is invalid. + """ + # Ensure that either is provided by the user. + if sdk_key is None and url is None: + raise optimizely_exceptions.InvalidInputException('Must provide at least one of sdk_key or url.') + + # Return URL if one is provided or use template and SDK key to get it. + if url is None: + try: + return url_template.format(sdk_key=sdk_key) + except (AttributeError, KeyError): + raise optimizely_exceptions.InvalidInputException( + 'Invalid url_template {} provided.'.format(url_template)) + + return url + + def set_update_interval(self, update_interval): + """ Helper method to set frequency at which datafile has to be polled and ProjectConfig updated. + + Args: + update_interval: Time in seconds after which to update datafile. + """ + self.update_interval = update_interval or enums.ConfigManager.DEFAULT_UPDATE_INTERVAL + + # If polling interval is less than minimum allowed interval then set it to default update interval. + if self.update_interval < enums.ConfigManager.MIN_UPDATE_INTERVAL: + self.logger.debug('Invalid update_interval {} provided. Defaulting to {}'.format( + update_interval, + enums.ConfigManager.DEFAULT_UPDATE_INTERVAL) + ) + self.update_interval = enums.ConfigManager.DEFAULT_UPDATE_INTERVAL + + def set_last_modified(self, response_headers): + """ Looks up and sets last modified time based on Last-Modified header in the response. + + Args: + response_headers: requests.Response.headers + """ + self.last_modified = response_headers.get(enums.HTTPHeaders.LAST_MODIFIED) + + def set_config(self, datafile): + """ Looks up and sets datafile and config based on response body. + + Args: + datafile: JSON string representing the Optimizely project. + """ + # TODO(ali): Add validation here to make sure that we do not update datafile and config if not a valid datafile. + self._datafile = datafile + # TODO(ali): Add notification listener. + self._config = project_config.ProjectConfig(self._datafile, self.logger, self.error_handler) + self.logger.debug('Received new datafile and updated config.') + + def get_config(self): + """ Returns instance of ProjectConfig. + + Returns: + ProjectConfig. + """ + return self._config + + def _handle_response(self, response): + """ Helper method to handle response containing datafile. + + Args: + response: requests.Response + """ + try: + response.raise_for_status() + except requests_exceptions.HTTPError as err: + self.logger.error('Fetching datafile from {} failed. Error: {}'.format(self.datafile_url, str(err))) + return + + # Leave datafile and config unchanged if it has not been modified. + if response.status_code == http_status_codes.not_modified: + self.logger.debug('Not updating config as datafile has not updated since {}.'.format(self.last_modified)) + return + + self.set_last_modified(response.headers) + self.set_config(response.content) + + def fetch_datafile(self): + """ Fetch datafile and set ProjectConfig. """ + + request_headers = {} + if self.last_modified: + request_headers[enums.HTTPHeaders.IF_MODIFIED_SINCE] = self.last_modified + + response = requests.get(self.datafile_url, + headers=request_headers, + timeout=enums.ConfigManager.REQUEST_TIMEOUT) + self._handle_response(response) + + @property + def is_running(self): + """ Check if polling thread is alive or not. """ + return self._polling_thread.is_alive() + + def _run(self): + """ Triggered as part of the thread which fetches the datafile and sleeps until next update interval. """ + while self.is_running: + self.fetch_datafile() + time.sleep(self.update_interval) + + def start(self): + """ Start the config manager and the thread to periodically fetch datafile. """ + if not self.is_running: + self._polling_thread.start() diff --git a/optimizely/helpers/enums.py b/optimizely/helpers/enums.py index 25f6da591..f86f584e4 100644 --- a/optimizely/helpers/enums.py +++ b/optimizely/helpers/enums.py @@ -36,6 +36,16 @@ class AudienceEvaluationLogs(object): 'newer release of the Optimizely SDK.' +class ConfigManager(object): + DATAFILE_URL_TEMPLATE = 'https://cdn.optimizely.com/datafiles/{sdk_key}.json' + # Default config update interval of 5 minutes + DEFAULT_UPDATE_INTERVAL = 5 * 60 + # Minimum config update interval of 1 second + MIN_UPDATE_INTERVAL = 1 + # Time in seconds before which request for datafile times out + REQUEST_TIMEOUT = 10 + + class ControlAttributes(object): BOT_FILTERING = '$opt_bot_filtering' BUCKETING_ID = '$opt_bucketing_id' @@ -79,6 +89,11 @@ class Errors(object): UNSUPPORTED_DATAFILE_VERSION = 'This version of the Python SDK does not support the given datafile version: "{}".' +class HTTPHeaders(object): + IF_MODIFIED_SINCE = 'If-Modified-Since' + LAST_MODIFIED = 'Last-Modified' + + class HTTPVerbs(object): GET = 'GET' POST = 'POST' diff --git a/optimizely/logger.py b/optimizely/logger.py index 293172074..9530b132e 100644 --- a/optimizely/logger.py +++ b/optimizely/logger.py @@ -1,4 +1,4 @@ -# Copyright 2016, Optimizely +# Copyright 2016, 2018-2019, Optimizely # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -46,8 +46,8 @@ def reset_logger(name, level=None, handler=None): handler.setFormatter(logging.Formatter(_DEFAULT_LOG_FORMAT)) # We don't use ``.addHandler``, since this logger may have already been - # instantiated elsewhere with a different handler. It should only ever - # have one, not many. + # instantiated elsewhere with a different handler. It should only ever + # have one, not many. logger.handlers = [handler] return logger diff --git a/optimizely/optimizely.py b/optimizely/optimizely.py index 0ca27fb6b..74f921acb 100644 --- a/optimizely/optimizely.py +++ b/optimizely/optimizely.py @@ -58,7 +58,7 @@ def __init__(self, except exceptions.InvalidInputException as error: self.is_valid = False # We actually want to log this error to stderr, so make sure the logger - # has a handler capable of doing that. + # has a handler capable of doing that. self.logger = _logging.reset_logger(self.logger_name) self.logger.exception(str(error)) return diff --git a/tests/test_config_manager.py b/tests/test_config_manager.py new file mode 100644 index 000000000..0d86b055e --- /dev/null +++ b/tests/test_config_manager.py @@ -0,0 +1,185 @@ +# Copyright 2019, Optimizely +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import mock +import requests +import unittest + +from optimizely import config_manager +from optimizely import exceptions as optimizely_exceptions +from optimizely import project_config +from optimizely.helpers import enums + + +class StaticConfigManagerTest(unittest.TestCase): + def test_get_config(self): + test_datafile = json.dumps({ + 'some_datafile_key': 'some_datafile_value', + 'version': project_config.SUPPORTED_VERSIONS[0] + }) + project_config_manager = config_manager.StaticConfigManager(datafile=test_datafile) + + # Assert that config is set. + self.assertIsInstance(project_config_manager.get_config(), project_config.ProjectConfig) + + +class PollingConfigManagerTest(unittest.TestCase): + def test_init__no_sdk_key_no_url__fails(self): + """ Test that initialization fails if there is no sdk_key or url provided. """ + self.assertRaisesRegexp(optimizely_exceptions.InvalidInputException, + 'Must provide at least one of sdk_key or url.', + config_manager.PollingConfigManager, sdk_key=None, url=None) + + def test_get_datafile_url__no_sdk_key_no_url_raises(self): + """ Test that get_datafile_url raises exception if no sdk_key or url is provided. """ + self.assertRaisesRegexp(optimizely_exceptions.InvalidInputException, + 'Must provide at least one of sdk_key or url.', + config_manager.PollingConfigManager.get_datafile_url, None, None, 'url_template') + + def test_get_datafile_url__invalid_url_template_raises(self): + """ Test that get_datafile_url raises if url_template is invalid. """ + # No url_template provided + self.assertRaisesRegexp(optimizely_exceptions.InvalidInputException, + 'Invalid url_template None provided', + config_manager.PollingConfigManager.get_datafile_url, 'optly_datafile_key', None, None) + + # Incorrect url_template provided + test_url_template = 'invalid_url_template_without_sdk_key_field_{key}' + self.assertRaisesRegexp(optimizely_exceptions.InvalidInputException, + 'Invalid url_template {} provided'.format(test_url_template), + config_manager.PollingConfigManager.get_datafile_url, + 'optly_datafile_key', None, test_url_template) + + def test_get_datafile_url__sdk_key_and_template_provided(self): + """ Test get_datafile_url when sdk_key and template are provided. """ + test_sdk_key = 'optly_key' + test_url_template = 'www.optimizelydatafiles.com/{sdk_key}.json' + expected_url = test_url_template.format(sdk_key=test_sdk_key) + self.assertEqual(expected_url, + config_manager.PollingConfigManager.get_datafile_url(test_sdk_key, None, test_url_template)) + + def test_get_datafile_url__url_and_template_provided(self): + """ Test get_datafile_url when url and url_template are provided. """ + test_url_template = 'www.optimizelydatafiles.com/{sdk_key}.json' + test_url = 'www.myoptimizelydatafiles.com/my_key.json' + self.assertEqual(test_url, config_manager.PollingConfigManager.get_datafile_url(None, + test_url, + test_url_template)) + + def test_get_datafile_url__sdk_key_and_url_and_template_provided(self): + """ Test get_datafile_url when sdk_key, url and url_template are provided. """ + test_sdk_key = 'optly_key' + test_url_template = 'www.optimizelydatafiles.com/{sdk_key}.json' + test_url = 'www.myoptimizelydatafiles.com/my_key.json' + + # Assert that if url is provided, it is always returned + self.assertEqual(test_url, config_manager.PollingConfigManager.get_datafile_url(test_sdk_key, + test_url, + test_url_template)) + + def test_set_update_interval(self): + project_config_manager = config_manager.PollingConfigManager(sdk_key='some_key') + + # Assert that update_interval cannot be set to less than allowed minimum and instead is set to default value. + project_config_manager.set_update_interval(0.42) + self.assertEqual(enums.ConfigManager.DEFAULT_UPDATE_INTERVAL, project_config_manager.update_interval) + + # Assert that if no update_interval is provided, it is set to default value. + project_config_manager.set_update_interval(None) + self.assertEqual(enums.ConfigManager.DEFAULT_UPDATE_INTERVAL, project_config_manager.update_interval) + + # Assert that if valid update_interval is provided, it is set to that value. + project_config_manager.set_update_interval(42) + self.assertEqual(42, project_config_manager.update_interval) + + def test_set_last_modified(self): + """ Test that set_last_modified sets last_modified field based on header. """ + project_config_manager = config_manager.PollingConfigManager(sdk_key='some_key') + self.assertIsNone(project_config_manager.last_modified) + + last_modified_time = 'Test Last Modified Time' + test_response_headers = { + 'Last-Modified': last_modified_time, + 'Some-Other-Important-Header': 'some_value' + } + project_config_manager.set_last_modified(test_response_headers) + self.assertEqual(last_modified_time, project_config_manager.last_modified) + + def test_set_and_get_config(self): + """ Test that set_last_modified sets config field based on datafile. """ + project_config_manager = config_manager.PollingConfigManager(sdk_key='some_key') + + # Assert that config is not set. + self.assertIsNone(project_config_manager.get_config()) + + # Set and check config. + project_config_manager.set_config(json.dumps({ + 'some_datafile_key': 'some_datafile_value', + 'version': project_config.SUPPORTED_VERSIONS[0] + })) + self.assertIsInstance(project_config_manager.get_config(), project_config.ProjectConfig) + + def test_fetch_datafile(self): + """ Test that fetch_datafile sets config and last_modified based on response. """ + project_config_manager = config_manager.PollingConfigManager(sdk_key='some_key') + expected_datafile_url = 'https://cdn.optimizely.com/datafiles/some_key.json' + test_headers = { + 'Last-Modified': 'New Time' + } + test_datafile = json.dumps({ + 'some_datafile_key': 'some_datafile_value', + 'version': project_config.SUPPORTED_VERSIONS[0] + }) + test_response = requests.Response() + test_response.status_code = 200 + test_response.headers = test_headers + test_response._content = test_datafile + with mock.patch('requests.get', return_value=test_response) as mock_requests: + project_config_manager.fetch_datafile() + + mock_requests.assert_called_once_with(expected_datafile_url, + headers={}, + timeout=enums.ConfigManager.REQUEST_TIMEOUT) + self.assertEqual(test_headers['Last-Modified'], project_config_manager.last_modified) + self.assertIsInstance(project_config_manager.get_config(), project_config.ProjectConfig) + + # Call fetch_datafile again and assert that request to URL is with If-Modified-Since header. + with mock.patch('requests.get', return_value=test_response) as mock_requests: + project_config_manager.fetch_datafile() + + mock_requests.assert_called_once_with(expected_datafile_url, + headers={'If-Modified-Since': test_headers['Last-Modified']}, + timeout=enums.ConfigManager.REQUEST_TIMEOUT) + self.assertEqual(test_headers['Last-Modified'], project_config_manager.last_modified) + self.assertIsInstance(project_config_manager.get_config(), project_config.ProjectConfig) + + def test_is_running(self): + """ Test is_running before and after starting thread. """ + project_config_manager = config_manager.PollingConfigManager(sdk_key='some_key') + self.assertFalse(project_config_manager.is_running) + with mock.patch('optimizely.config_manager.PollingConfigManager.fetch_datafile') as mock_fetch_datafile: + project_config_manager.start() + self.assertTrue(project_config_manager.is_running) + + mock_fetch_datafile.assert_called_with() + + def test_start(self): + """ Test that calling start starts the polling thread. """ + project_config_manager = config_manager.PollingConfigManager(sdk_key='some_key') + self.assertFalse(project_config_manager._polling_thread.is_alive()) + with mock.patch('optimizely.config_manager.PollingConfigManager.fetch_datafile') as mock_fetch_datafile: + project_config_manager.start() + self.assertTrue(project_config_manager._polling_thread.is_alive()) + + mock_fetch_datafile.assert_called_with()