diff --git a/compose/cli/command.py b/compose/cli/command.py index 67b77f31b57..d21bc85441d 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -4,7 +4,6 @@ import logging import os import re -import yaml import six from ..project import Project @@ -14,6 +13,7 @@ from .docker_client import docker_client from . import verbose_proxy from . import errors +from .config import from_yaml_with_environment_vars from .. import __version__ log = logging.getLogger(__name__) @@ -70,11 +70,17 @@ def get_client(self, verbose=False): return client def get_config(self, config_path): - try: - with open(config_path, 'r') as fh: - return yaml.safe_load(fh) - except IOError as e: - raise errors.UserError(six.text_type(e)) + """ + Access a :class:fig.cli.config.Config object from string representing a file location, + returned as a dict with any variables appropriately interpolated + + :param config_path: the full path, including filename where fig.yml lives + :type config_path: str + + :return: dict + :rtype: dict + """ + return from_yaml_with_environment_vars(config_path) def get_project(self, config_path, project_name=None, verbose=False): try: diff --git a/compose/cli/config.py b/compose/cli/config.py new file mode 100644 index 00000000000..b0187811c13 --- /dev/null +++ b/compose/cli/config.py @@ -0,0 +1,86 @@ +from __future__ import unicode_literals +from __future__ import absolute_import + +import six +import errno +import yaml +import os +import re + +from compose.cli import errors + + +def resolve_environment_vars(value): + """ + Matches our environment variable pattern, replaces the value with the value in the env + Supports multiple variables per line + + A future improvement here would be to also reference a separate dictionary that keeps track of + configurable variables via flat file. + + :param value: any value that is reachable by a key in a dict represented by a yaml file + :return: the value itself if not a string with an environment variable, otherwise the value specified in the env + """ + if not isinstance(value, six.string_types): + return value + + # First, identify any variables + env_regex = re.compile(r'([^\\])?(?P\$\{[^\}]+\})') + + split_string = re.split(env_regex, value) + result_string = '' + + instance_regex = r'^\$\{(?P[^\}^:]+)(:(?P[^\}]+))?\}$' + for split_value in split_string: + if not split_value: + continue + + match_object = re.match(instance_regex, split_value) + if not match_object: + result_string += split_value + continue + + result = os.environ.get(match_object.group('env_var'), match_object.group('default_val')) + + if result is None: + raise errors.UserError("No value for ${%s} found in environment." % (match_object.group('env_var')) + + "Please set a value in the environment or provide a default.") + + result_string += re.sub(instance_regex, result, split_value) + + return result_string + + +def with_environment_vars(value): + """ + Recursively interpolates environment variables for a structured or unstructured value + + :param value: a dict, list, or any other kind of value + + :return: the dict with its values interpolated from the env + :rtype: dict + """ + if type(value) == dict: + return dict([(subkey, with_environment_vars(subvalue)) + for subkey, subvalue in value.items()]) + elif type(value) == list: + return [resolve_environment_vars(x) for x in value] + else: + return resolve_environment_vars(value) + + +def from_yaml_with_environment_vars(yaml_filename): + """ + Resolves environment variables in values defined by a YAML file and transformed into a dict + :param yaml_filename: the name of the yaml file + :type yaml_filename: str + + :return: a dict with environment variables properly interpolated + """ + try: + with open(yaml_filename, 'r') as fh: + return with_environment_vars(yaml.safe_load(fh)) + except IOError as e: + if e.errno == errno.ENOENT: + raise errors.FigFileNotFound(os.path.basename(e.filename)) + raise errors.UserError(six.text_type(e)) diff --git a/docs/yml.md b/docs/yml.md index 228fe2ce517..fa0023f7204 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -16,6 +16,14 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. +Fig supports using environment variables with optional defaults as values. The accepted patterns are as follows: + +``` +${ENVIRONMENT_VARIABLE:default} +or simply +${ENVIRONMENT_VARIABLE} +``` + ### image Tag or partial image ID. Can be local or remote - Compose will attempt to @@ -25,6 +33,7 @@ pull if it doesn't exist locally. image: ubuntu image: orchardup/postgresql image: a4bc65fd +image: ${SPECIFIED_OS:ubuntu} ``` ### build @@ -34,6 +43,8 @@ with a generated name, and use that image thereafter. ``` build: /path/to/build/dir +or +build: ${BUILD_PATH:/path/to/build/dir} ``` ### command @@ -42,6 +53,7 @@ Override the default command. ``` command: bundle exec thin -p 3000 +command: RACK_ENV=${RACK_ENV:production} bundle exec rackup -p 80 ``` @@ -56,6 +68,7 @@ links: - db - db:database - redis + - ${SEARCH_SERVICE:solr}:search ``` An entry with the alias' name will be created in `/etc/hosts` inside containers @@ -65,6 +78,7 @@ for this service, e.g: 172.17.2.186 db 172.17.2.186 database 172.17.2.187 redis +172.17.2.188 search ``` Environment variables will also be created - see the [environment variable @@ -100,6 +114,7 @@ ports: - "8000:8000" - "49100:22" - "127.0.0.1:8001:8001" + - ${DEFAULT_PORT:"80:800"} ``` ### expose @@ -111,6 +126,7 @@ accessible to linked services. Only the internal port can be specified. expose: - "3000" - "8000" + - ${EXPOSE_PORT} ``` ### volumes @@ -123,6 +139,7 @@ volumes: - /var/lib/mysql - cache/:/tmp/cache - ~/configs:/etc/configs/:ro + - ${JAVA_HOME:/usr/lib/jvm/jdk1.7.0/}/lib ``` ### volumes_from @@ -133,6 +150,7 @@ Mount all of the volumes from another service or container. volumes_from: - service_name - container_name + - ${SELECTED_SERVICE_VOLUME:service_name} ``` ### environment @@ -145,10 +163,12 @@ machine Compose is running on, which can be helpful for secret or host-specific ``` environment: RACK_ENV: development + LOGLEVEL: ${APP_LOGLEVEL:INFO} SESSION_SECRET: environment: - RACK_ENV=development + - LOGLEVEL=${APP_LOGLEVEL:INFO} - SESSION_SECRET ``` @@ -176,6 +196,7 @@ net: "bridge" net: "none" net: "container:[name or id]" net: "host" +net: ${NET_MODE:"bridge"} ``` ### dns @@ -184,6 +205,7 @@ Custom DNS servers. Can be a single value or a list. ``` dns: 8.8.8.8 +dns: ${DNS_SERVER:8.8.8.8} dns: - 8.8.8.8 - 9.9.9.9 @@ -218,6 +240,7 @@ dns_search: Each of these is a single value, analogous to its [docker run](https://docs.docker.com/reference/run/) counterpart. +Each entry can use environment variables. ``` cpu_shares: 73 diff --git a/tests/unit/cli/config_test.py b/tests/unit/cli/config_test.py new file mode 100644 index 00000000000..d1c6a654dd0 --- /dev/null +++ b/tests/unit/cli/config_test.py @@ -0,0 +1,141 @@ +from __future__ import unicode_literals +from __future__ import absolute_import +from tests import unittest + +from compose.cli import config +from compose.cli.errors import UserError + +import yaml +import os + + +TEST_YML_DICT_WITH_DEFAULTS = yaml.load("""db: + image: postgres +web: + build: ${DJANGO_BUILD:.} + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + environment: + DJANGO_ENV: ${TEST_VALUE:production} + ports: + - ${DJANGO_PORT:"8000:8000"} + - ${PORT_A:80}:${PORT_B:80} + links: + - db""") + +TEST_YML_DICT_NO_DEFAULTS = yaml.load("""db: + image: postgres +web: + build: ${TEST_VALUE} + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + links: + - db""") + + +class ConfigTestCase(unittest.TestCase): + + def setUp(self): + self.environment_variables = { + "TEST_VALUE": os.environ.get('TEST_VALUE'), + "PORT_A": os.environ.get('PORT_A'), + "PORT_B": os.environ.get('PORT_B') + } + + def tearDown(self): + for variable, former_value in self.environment_variables.items(): + if former_value is not None: + os.environ[variable] = former_value + elif variable in os.environ: + del os.environ[variable] + + def test_with_resolve_environment_vars_nonmatch_values(self): + # It should just return non-string values + self.assertEqual(1, config.resolve_environment_vars(1)) + self.assertEqual([], config.resolve_environment_vars([])) + self.assertEqual({}, config.resolve_environment_vars({})) + self.assertEqual(1.234, config.resolve_environment_vars(1.234)) + self.assertEqual(None, config.resolve_environment_vars(None)) + + # Any string that doesn't match our regex should just be returned + self.assertEqual('localhost', config.resolve_environment_vars('localhost')) + + expected = "some-other-host" + os.environ['TEST_VALUE'] = expected + + # Bare mentions of the environment variable shouldn't work + value = 'TEST_VALUE:foo' + self.assertEqual(value, config.resolve_environment_vars(value)) + + value = 'TEST_VALUE' + self.assertEqual(value, config.resolve_environment_vars(value)) + + # Incomplete pattern shouldn't work as well + for value in ['${TEST_VALUE', '$TEST_VALUE', '{TEST_VALUE}']: + self.assertEqual(value, config.resolve_environment_vars(value)) + value += ':foo' + self.assertEqual(value, config.resolve_environment_vars(value)) + + def test_fully_interpolated_matches(self): + expected = "some-other-host" + os.environ['TEST_VALUE'] = expected + + # if we have a basic match + self.assertEqual(expected, config.resolve_environment_vars("${TEST_VALUE}")) + + # if we have a match with a default value + self.assertEqual(expected, config.resolve_environment_vars("${TEST_VALUE:localhost}")) + + # escaping should prevent interpolation + escaped_no_default = "\${TEST_VALUE}" + escaped_with_default = "\${TEST_VALUE:localhost}" + self.assertEqual(escaped_no_default, escaped_no_default) + self.assertEqual(escaped_with_default, escaped_with_default) + + # if we have no match but a default value + del os.environ['TEST_VALUE'] + self.assertEqual('localhost', config.resolve_environment_vars("${TEST_VALUE:localhost}")) + + def test_fully_interpolated_errors(self): + if 'TEST_VALUE' in os.environ: + del os.environ['TEST_VALUE'] + self.assertRaises(UserError, config.resolve_environment_vars, "${TEST_VALUE}") + + def test_functional_defaults_as_dict(self): + d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) + + # tests the basic structure and functionality of defaults + self.assertEqual(d['web']['build'], '.') + + # test that environment variables with defaults are handled in lists + self.assertEqual(d['web']['ports'][0], '"8000:8000"') + + # test that environment variables with defaults are handled more than once in the same line + self.assertEqual(d['web']['ports'][1], '80:80') + + # test that environment variables with defaults are handled with variables more than once in the same line + os.environ['PORT_A'] = '8080' + os.environ['PORT_B'] = '9000' + d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) + self.assertEqual(d['web']['ports'][1], '8080:9000') + + # test that environment variables with defaults are handled in dictionaries + self.assertEqual(d['web']['environment']['DJANGO_ENV'], 'production') + + # test that having an environment variable set properly pulls it + os.environ['TEST_VALUE'] = 'development' + d = config.with_environment_vars(TEST_YML_DICT_WITH_DEFAULTS) + self.assertEqual(d['web']['environment']['DJANGO_ENV'], 'development') + + def test_functional_no_defaults(self): + # test that not having defaults raises an error in a real YML situation + self.assertRaises(UserError, config.with_environment_vars, TEST_YML_DICT_NO_DEFAULTS) + + # test that a bare environment variable is interpolated + # note that we have to reload it + os.environ['TEST_VALUE'] = '/home/ubuntu/django' + self.assertEqual(config.with_environment_vars(TEST_YML_DICT_NO_DEFAULTS)['web']['build'], '/home/ubuntu/django') \ No newline at end of file