Add support for environment variables with default values in fig.yml #845
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| from __future__ import unicode_literals | ||
| from __future__ import absolute_import | ||
|
|
||
| import six | ||
| import errno | ||
| import yaml | ||
| import os | ||
| import re | ||
|
|
||
| from compose.cli import errors | ||
|
|
||
|
|
||
| def resolve_environment_vars(value): | ||
| """ | ||
| Matches our environment variable pattern, replaces the value with the value in the env | ||
| Supports multiple variables per line | ||
|
|
||
| A future improvement here would be to also reference a separate dictionary that keeps track of | ||
| configurable variables via flat file. | ||
|
|
||
| :param value: any value that is reachable by a key in a dict represented by a yaml file | ||
| :return: the value itself if not a string with an environment variable, otherwise the value specified in the env | ||
| """ | ||
| if not isinstance(value, six.string_types): | ||
| return value | ||
|
|
||
| # First, identify any variables | ||
| env_regex = re.compile(r'([^\\])?(?P<variable>\$\{[^\}]+\})') | ||
|
|
||
| split_string = re.split(env_regex, value) | ||
| result_string = '' | ||
|
|
||
| instance_regex = r'^\$\{(?P<env_var>[^\}^:]+)(:(?P<default_val>[^\}]+))?\}$' | ||
| for split_value in split_string: | ||
| if not split_value: | ||
| continue | ||
|
|
||
| match_object = re.match(instance_regex, split_value) | ||
| if not match_object: | ||
| result_string += split_value | ||
| continue | ||
|
|
||
| result = os.environ.get(match_object.group('env_var'), match_object.group('default_val')) | ||
|
|
||
| if result is None: | ||
| raise errors.UserError("No value for ${%s} found in environment." % (match_object.group('env_var')) | ||
| + "Please set a value in the environment or provide a default.") | ||
|
|
||
| result_string += re.sub(instance_regex, result, split_value) | ||
|
|
||
| return result_string | ||
|
|
||
|
|
||
| def with_environment_vars(value): | ||
| """ | ||
| Recursively interpolates environment variables for a structured or unstructured value | ||
|
|
||
| :param value: a dict, list, or any other kind of value | ||
|
|
||
| :return: the dict with its values interpolated from the env | ||
| :rtype: dict | ||
| """ | ||
| if type(value) == dict: | ||
| return dict([(subkey, with_environment_vars(subvalue)) | ||
| for subkey, subvalue in value.items()]) | ||
| elif type(value) == list: | ||
| return [resolve_environment_vars(x) for x in value] | ||
| else: | ||
| return resolve_environment_vars(value) | ||
|
|
||
|
|
||
| def from_yaml_with_environment_vars(yaml_filename): | ||
| """ | ||
| Resolves environment variables in values defined by a YAML file and transformed into a dict | ||
| :param yaml_filename: the name of the yaml file | ||
| :type yaml_filename: str | ||
|
|
||
| :return: a dict with environment variables properly interpolated | ||
| """ | ||
| try: | ||
| with open(yaml_filename, 'r') as fh: | ||
| return with_environment_vars(yaml.safe_load(fh)) | ||
| except IOError as e: | ||
| if e.errno == errno.ENOENT: | ||
| raise errors.FigFileNotFound(os.path.basename(e.filename)) | ||
| raise errors.UserError(six.text_type(e)) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| from __future__ import unicode_literals | ||
| from __future__ import absolute_import | ||
| from tests import unittest | ||
|
|
||
| from compose.cli import config | ||
| from compose.cli.errors import UserError | ||
|
|
||
| import yaml | ||
| import os | ||
|
|
||
|
|
||
| TEST_YML_DICT_WITH_DEFAULTS = yaml.load("""db: | ||
| image: postgres | ||
| web: | ||
| build: ${DJANGO_BUILD:.} | ||
| command: python manage.py runserver 0.0.0.0:8000 | ||
| volumes: | ||
| - .:/code | ||
| environment: | ||
| DJANGO_ENV: ${TEST_VALUE:production} | ||
| ports: | ||
| - ${DJANGO_PORT:"8000:8000"} | ||
| - ${PORT_A:80}:${PORT_B:80} | ||
| links: | ||
| - db""") | ||
|
|
||
| TEST_YML_DICT_NO_DEFAULTS = yaml.load("""db: | ||
| image: postgres | ||
| web: | ||
| build: ${TEST_VALUE} | ||
| command: python manage.py runserver 0.0.0.0:8000 | ||
| volumes: | ||
| - .:/code | ||
| ports: | ||
| - "8000:8000" | ||
| links: | ||
| - db""") | ||
|
|
||
|
|
||
| class ConfigTestCase(unittest.TestCase): | ||
|
|
||
| def setUp(self): | ||
| self.environment_variables = { | ||
| "TEST_VALUE": os.environ.get('TEST_VALUE'), | ||
| "PORT_A": os.environ.get('PORT_A'), | ||
| "PORT_B": os.environ.get('PORT_B') | ||
| } | ||
|
|
||
| def tearDown(self): | ||
| for variable, former_value in self.environment_variables.items(): | ||
| if former_value is not None: | ||
| os.environ[variable] = former_value | ||
| elif variable in os.environ: | ||
| del os.environ[variable] | ||
|
|
||
| def test_with_resolve_environment_vars_nonmatch_values(self): | ||
| # It should just return non-string values | ||
| self.assertEqual(1, config.resolve_environment_vars(1)) | ||
| self.assertEqual([], config.resolve_environment_vars([])) | ||
| self.assertEqual({}, config.resolve_environment_vars({})) | ||
| self.assertEqual(1.234, config.resolve_environment_vars(1.234)) | ||
| self.assertEqual(None, config.resolve_environment_vars(None)) | ||
|
|
||
| # Any string that doesn't match our regex should just be returned | ||
| self.assertEqual('localhost', config.resolve_environment_vars('localhost')) | ||
|
|
||
| expected = "some-other-host" | ||
| os.environ['TEST_VALUE'] = expected | ||
|
|
||
| # Bare mentions of the environment variable shouldn't work | ||
| value = 'TEST_VALUE:foo' | ||
| self.assertEqual(value, config.resolve_environment_vars(value)) | ||
|
|
||
| value = 'TEST_VALUE' | ||
| self.assertEqual(value, config.resolve_environment_vars(value)) | ||
|
|
||
| # Incomplete pattern shouldn't work as well | ||
| for value in ['${TEST_VALUE', '$TEST_VALUE', '{TEST_VALUE}']: | ||
| self.assertEqual(value, config.resolve_environment_vars(value)) | ||
| value += ':foo' | ||
| self.assertEqual(value, config.resolve_environment_vars(value)) | ||
|
|
||
| def test_fully_interpolated_matches(self): | ||
| expected = "some-other-host" | ||
| os.environ['TEST_VALUE'] = expected | ||
|
|
||
| # if we have a basic match | ||
| self.assertEqual(expected, config.resolve_environment_vars("${TEST_VALUE}")) | ||
|
|
||
| # if we have a match with a default value | ||
| self.assertEqual(expected, config.resolve_environment_vars("${TEST_VALUE:localhost}")) | ||
|
|
||
| # escaping should prevent interpolation | ||
| escaped_no_default = "\${TEST_VALUE}" | ||
| escaped_with_default = "\${TEST_VALUE:localhost}" | ||
| self.assertEqual(escaped_no_default, escaped_no_default) | ||
| self.assertEqual(escaped_with_default, escaped_with_default) | ||
|
|
||
| # if we have no match but a default value | ||
| del os.environ['TEST_VALUE'] | ||
| self.assertEqual('localhost', config.resolve_environment_vars("${TEST_VALUE:localhost}")) | ||
|
|
||
| def test_fully_interpolated_errors(self): | ||
| if 'TEST_VALUE' in os.environ: | ||
| del os.environ['TEST_VALUE'] | ||
| self.assertRaises(UserError, config.resolve_environment_vars, "${TEST_VALUE}") | ||
|
|
||
| def test_functional_defaults_as_dict(self): | ||
| d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) | ||
|
|
||
| # tests the basic structure and functionality of defaults | ||
| self.assertEqual(d['web']['build'], '.') | ||
|
|
||
|
|
||
| # test that environment variables with defaults are handled in lists | ||
| self.assertEqual(d['web']['ports'][0], '"8000:8000"') | ||
|
|
||
| # test that environment variables with defaults are handled more than once in the same line | ||
| self.assertEqual(d['web']['ports'][1], '80:80') | ||
|
|
||
| # test that environment variables with defaults are handled with variables more than once in the same line | ||
| os.environ['PORT_A'] = '8080' | ||
| os.environ['PORT_B'] = '9000' | ||
| d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) | ||
| self.assertEqual(d['web']['ports'][1], '8080:9000') | ||
|
|
||
| # test that environment variables with defaults are handled in dictionaries | ||
| self.assertEqual(d['web']['environment']['DJANGO_ENV'], 'production') | ||
|
|
||
| # test that having an environment variable set properly pulls it | ||
| os.environ['TEST_VALUE'] = 'development' | ||
| d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) | ||
| self.assertEqual(d['web']['environment']['DJANGO_ENV'], 'development') | ||
|
|
||
| def test_functional_no_defaults(self): | ||
| # test that not having defaults raises an error in a real YML situation | ||
| self.assertRaises(UserError, config.with_environment_vars, TEST_YML_DICT_NO_DEFAULTS) | ||
|
|
||
| # test that a bare environment variable is interpolated | ||
| # note that we have to reload it | ||
| os.environ['TEST_VALUE'] = '/home/ubuntu/django' | ||
| self.assertEqual(config.with_environment_vars(TEST_YML_DICT_NO_DEFAULTS)['web']['build'], '/home/ubuntu/django') | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Interested to see a test case and/or instructions for providing port numbers <= 60 via environment variables, to test for this: https://github.com/docker/fig/blob/master/docs/yml.md#ports. Don't know if it's possible to do, but thinking of:
How should that be done to prevent fig from interpreting them incorrectly?