From 061460d2d972f6b4573d59f41cccd8231f95f27b Mon Sep 17 00:00:00 2001 From: Robert Elwell Date: Wed, 14 Jan 2015 15:40:34 -0800 Subject: [PATCH] Add support for environment variables with backoff values in fig.yml This change resolves #495 by introducing a Config class inside the cli library. The Config class is responsible for loading a .yml file and evaluating its contents for environment variables. The Config immediately reloads the data as a dictionary, which reduces compatibility issues within previously written code. This behavior also prevents race conditions caused by changes to the environment during long-running fig builds. This change is valuable because it allows for a greater degree of automation and variability when working with multiple environments or deployment types that use identically built containers. We currently have to define separate services for each degree of variability we want to account for. Future steps for this change may include introducing template files, or adding access and manipulation APIs directly so that we can perpetuate the config class everywhere we are currently using bare dictionaries. Added in revision requests from @dnephin: * Terminology fix in docs/yml.md * Refactored config class into a series of functions * Updated tests to reflect functional interpolation * Improved forward compatibility Added in revision requests from @thaJeztah * Terminology fixes Added in revision requests from @aanand * Support for escaping Added in revision requests from @thaJeztah * Support for multiple variables per line Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. Signed-off-by: Robert Elwell Sponsored by: Lookout, Inc. --- compose/cli/command.py | 18 +++-- compose/cli/config.py | 86 +++++++++++++++++++++ docs/yml.md | 23 ++++++ tests/unit/cli/config_test.py | 141 ++++++++++++++++++++++++++++++++++ 4 files changed, 262 insertions(+), 6 deletions(-) create mode 100644 compose/cli/config.py create mode 100644 tests/unit/cli/config_test.py diff --git a/compose/cli/command.py b/compose/cli/command.py index 67b77f31b57..d21bc85441d 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -4,7 +4,6 @@ import logging import os import re -import yaml import six from ..project import Project @@ -14,6 +13,7 @@ from .docker_client import docker_client from . import verbose_proxy from . import errors +from .config import from_yaml_with_environment_vars from .. import __version__ log = logging.getLogger(__name__) @@ -70,11 +70,17 @@ def get_client(self, verbose=False): return client def get_config(self, config_path): - try: - with open(config_path, 'r') as fh: - return yaml.safe_load(fh) - except IOError as e: - raise errors.UserError(six.text_type(e)) + """ + Access a :class:fig.cli.config.Config object from string representing a file location, + returned as a dict with any variables appropriately interpolated + + :param config_path: the full path, including filename where fig.yml lives + :type config_path: str + + :return: dict + :rtype: dict + """ + return from_yaml_with_environment_vars(config_path) def get_project(self, config_path, project_name=None, verbose=False): try: diff --git a/compose/cli/config.py b/compose/cli/config.py new file mode 100644 index 00000000000..b0187811c13 --- /dev/null +++ b/compose/cli/config.py @@ -0,0 +1,86 @@ +from __future__ import unicode_literals +from __future__ import absolute_import + +import six +import errno +import yaml +import os +import re + +from compose.cli import errors + + +def resolve_environment_vars(value): + """ + Matches our environment variable pattern, replaces the value with the value in the env + Supports multiple variables per line + + A future improvement here would be to also reference a separate dictionary that keeps track of + configurable variables via flat file. + + :param value: any value that is reachable by a key in a dict represented by a yaml file + :return: the value itself if not a string with an environment variable, otherwise the value specified in the env + """ + if not isinstance(value, six.string_types): + return value + + # First, identify any variables + env_regex = re.compile(r'([^\\])?(?P\$\{[^\}]+\})') + + split_string = re.split(env_regex, value) + result_string = '' + + instance_regex = r'^\$\{(?P[^\}^:]+)(:(?P[^\}]+))?\}$' + for split_value in split_string: + if not split_value: + continue + + match_object = re.match(instance_regex, split_value) + if not match_object: + result_string += split_value + continue + + result = os.environ.get(match_object.group('env_var'), match_object.group('default_val')) + + if result is None: + raise errors.UserError("No value for ${%s} found in environment." % (match_object.group('env_var')) + + "Please set a value in the environment or provide a default.") + + result_string += re.sub(instance_regex, result, split_value) + + return result_string + + +def with_environment_vars(value): + """ + Recursively interpolates environment variables for a structured or unstructured value + + :param value: a dict, list, or any other kind of value + + :return: the dict with its values interpolated from the env + :rtype: dict + """ + if type(value) == dict: + return dict([(subkey, with_environment_vars(subvalue)) + for subkey, subvalue in value.items()]) + elif type(value) == list: + return [resolve_environment_vars(x) for x in value] + else: + return resolve_environment_vars(value) + + +def from_yaml_with_environment_vars(yaml_filename): + """ + Resolves environment variables in values defined by a YAML file and transformed into a dict + :param yaml_filename: the name of the yaml file + :type yaml_filename: str + + :return: a dict with environment variables properly interpolated + """ + try: + with open(yaml_filename, 'r') as fh: + return with_environment_vars(yaml.safe_load(fh)) + except IOError as e: + if e.errno == errno.ENOENT: + raise errors.FigFileNotFound(os.path.basename(e.filename)) + raise errors.UserError(six.text_type(e)) diff --git a/docs/yml.md b/docs/yml.md index 228fe2ce517..fa0023f7204 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -16,6 +16,14 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. +Fig supports using environment variables with optional defaults as values. The accepted patterns are as follows: + +``` +${ENVIRONMENT_VARIABLE:default} +or simply +${ENVIRONMENT_VARIABLE} +``` + ### image Tag or partial image ID. Can be local or remote - Compose will attempt to @@ -25,6 +33,7 @@ pull if it doesn't exist locally. image: ubuntu image: orchardup/postgresql image: a4bc65fd +image: ${SPECIFIED_OS:ubuntu} ``` ### build @@ -34,6 +43,8 @@ with a generated name, and use that image thereafter. ``` build: /path/to/build/dir +or +build: ${BUILD_PATH:/path/to/build/dir} ``` ### command @@ -42,6 +53,7 @@ Override the default command. ``` command: bundle exec thin -p 3000 +command: RACK_ENV=${RACK_ENV:production} bundle exec rackup -p 80 ``` @@ -56,6 +68,7 @@ links: - db - db:database - redis + - ${SEARCH_SERVICE:solr}:search ``` An entry with the alias' name will be created in `/etc/hosts` inside containers @@ -65,6 +78,7 @@ for this service, e.g: 172.17.2.186 db 172.17.2.186 database 172.17.2.187 redis +172.17.2.188 search ``` Environment variables will also be created - see the [environment variable @@ -100,6 +114,7 @@ ports: - "8000:8000" - "49100:22" - "127.0.0.1:8001:8001" + - ${DEFAULT_PORT:"80:800"} ``` ### expose @@ -111,6 +126,7 @@ accessible to linked services. Only the internal port can be specified. expose: - "3000" - "8000" + - ${EXPOSE_PORT} ``` ### volumes @@ -123,6 +139,7 @@ volumes: - /var/lib/mysql - cache/:/tmp/cache - ~/configs:/etc/configs/:ro + - ${JAVA_HOME:/usr/lib/jvm/jdk1.7.0/}/lib ``` ### volumes_from @@ -133,6 +150,7 @@ Mount all of the volumes from another service or container. volumes_from: - service_name - container_name + - ${SELECTED_SERVICE_VOLUME:service_name} ``` ### environment @@ -145,10 +163,12 @@ machine Compose is running on, which can be helpful for secret or host-specific ``` environment: RACK_ENV: development + LOGLEVEL: ${APP_LOGLEVEL:INFO} SESSION_SECRET: environment: - RACK_ENV=development + - LOGLEVEL=${APP_LOGLEVEL:INFO} - SESSION_SECRET ``` @@ -176,6 +196,7 @@ net: "bridge" net: "none" net: "container:[name or id]" net: "host" +net: ${NET_MODE:"bridge"} ``` ### dns @@ -184,6 +205,7 @@ Custom DNS servers. Can be a single value or a list. ``` dns: 8.8.8.8 +dns: ${DNS_SERVER:8.8.8.8} dns: - 8.8.8.8 - 9.9.9.9 @@ -218,6 +240,7 @@ dns_search: Each of these is a single value, analogous to its [docker run](https://docs.docker.com/reference/run/) counterpart. +Each entry can use environment variables. ``` cpu_shares: 73 diff --git a/tests/unit/cli/config_test.py b/tests/unit/cli/config_test.py new file mode 100644 index 00000000000..d1c6a654dd0 --- /dev/null +++ b/tests/unit/cli/config_test.py @@ -0,0 +1,141 @@ +from __future__ import unicode_literals +from __future__ import absolute_import +from tests import unittest + +from compose.cli import config +from compose.cli.errors import UserError + +import yaml +import os + + +TEST_YML_DICT_WITH_DEFAULTS = yaml.load("""db: + image: postgres +web: + build: ${DJANGO_BUILD:.} + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + environment: + DJANGO_ENV: ${TEST_VALUE:production} + ports: + - ${DJANGO_PORT:"8000:8000"} + - ${PORT_A:80}:${PORT_B:80} + links: + - db""") + +TEST_YML_DICT_NO_DEFAULTS = yaml.load("""db: + image: postgres +web: + build: ${TEST_VALUE} + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + links: + - db""") + + +class ConfigTestCase(unittest.TestCase): + + def setUp(self): + self.environment_variables = { + "TEST_VALUE": os.environ.get('TEST_VALUE'), + "PORT_A": os.environ.get('PORT_A'), + "PORT_B": os.environ.get('PORT_B') + } + + def tearDown(self): + for variable, former_value in self.environment_variables.items(): + if former_value is not None: + os.environ[variable] = former_value + elif variable in os.environ: + del os.environ[variable] + + def test_with_resolve_environment_vars_nonmatch_values(self): + # It should just return non-string values + self.assertEqual(1, config.resolve_environment_vars(1)) + self.assertEqual([], config.resolve_environment_vars([])) + self.assertEqual({}, config.resolve_environment_vars({})) + self.assertEqual(1.234, config.resolve_environment_vars(1.234)) + self.assertEqual(None, config.resolve_environment_vars(None)) + + # Any string that doesn't match our regex should just be returned + self.assertEqual('localhost', config.resolve_environment_vars('localhost')) + + expected = "some-other-host" + os.environ['TEST_VALUE'] = expected + + # Bare mentions of the environment variable shouldn't work + value = 'TEST_VALUE:foo' + self.assertEqual(value, config.resolve_environment_vars(value)) + + value = 'TEST_VALUE' + self.assertEqual(value, config.resolve_environment_vars(value)) + + # Incomplete pattern shouldn't work as well + for value in ['${TEST_VALUE', '$TEST_VALUE', '{TEST_VALUE}']: + self.assertEqual(value, config.resolve_environment_vars(value)) + value += ':foo' + self.assertEqual(value, config.resolve_environment_vars(value)) + + def test_fully_interpolated_matches(self): + expected = "some-other-host" + os.environ['TEST_VALUE'] = expected + + # if we have a basic match + self.assertEqual(expected, config.resolve_environment_vars("${TEST_VALUE}")) + + # if we have a match with a default value + self.assertEqual(expected, config.resolve_environment_vars("${TEST_VALUE:localhost}")) + + # escaping should prevent interpolation + escaped_no_default = "\${TEST_VALUE}" + escaped_with_default = "\${TEST_VALUE:localhost}" + self.assertEqual(escaped_no_default, escaped_no_default) + self.assertEqual(escaped_with_default, escaped_with_default) + + # if we have no match but a default value + del os.environ['TEST_VALUE'] + self.assertEqual('localhost', config.resolve_environment_vars("${TEST_VALUE:localhost}")) + + def test_fully_interpolated_errors(self): + if 'TEST_VALUE' in os.environ: + del os.environ['TEST_VALUE'] + self.assertRaises(UserError, config.resolve_environment_vars, "${TEST_VALUE}") + + def test_functional_defaults_as_dict(self): + d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) + + # tests the basic structure and functionality of defaults + self.assertEqual(d['web']['build'], '.') + + # test that environment variables with defaults are handled in lists + self.assertEqual(d['web']['ports'][0], '"8000:8000"') + + # test that environment variables with defaults are handled more than once in the same line + self.assertEqual(d['web']['ports'][1], '80:80') + + # test that environment variables with defaults are handled with variables more than once in the same line + os.environ['PORT_A'] = '8080' + os.environ['PORT_B'] = '9000' + d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) + self.assertEqual(d['web']['ports'][1], '8080:9000') + + # test that environment variables with defaults are handled in dictionaries + self.assertEqual(d['web']['environment']['DJANGO_ENV'], 'production') + + # test that having an environment variable set properly pulls it + os.environ['TEST_VALUE'] = 'development' + d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) + self.assertEqual(d['web']['environment']['DJANGO_ENV'], 'development') + + def test_functional_no_defaults(self): + # test that not having defaults raises an error in a real YML situation + self.assertRaises(UserError, config.with_environment_vars, TEST_YML_DICT_NO_DEFAULTS) + + # test that a bare environment variable is interpolated + # note that we have to reload it + os.environ['TEST_VALUE'] = '/home/ubuntu/django' + self.assertEqual(config.with_environment_vars(TEST_YML_DICT_NO_DEFAULTS)['web']['build'], '/home/ubuntu/django') \ No newline at end of file