diff --git a/dmake/deepobuild.py b/dmake/deepobuild.py index 2dcbaf97..0a4ba834 100644 --- a/dmake/deepobuild.py +++ b/dmake/deepobuild.py @@ -6,7 +6,7 @@ import re import requests.exceptions from string import Template -from dmake.serializer import ValidationError, FieldSerializer, YAML2PipelineSerializer +from dmake.serializer import ValidationError, FieldSerializer, YAML2PipelineSerializer, SerializerMixin import dmake.common as common from dmake.common import DMakeException, SharedVolumeNotFoundException, append_command import dmake.kubernetes as k8s_utils @@ -392,14 +392,43 @@ class HTMLReportSerializer(YAML2PipelineSerializer): index = FieldSerializer("string", default = "index.html", help_text = "Main page.") title = FieldSerializer("string", default = "HTML Report", help_text = "Main page title.") -class DockerLinkSerializer(YAML2PipelineSerializer): +class ProbePortMixin(SerializerMixin): + probe_ports = FieldSerializer(["string", "array"], default = "auto", child = "string", help_text = "Either 'none', 'auto' or a list of ports in the form 1234/tcp or 1234/udp") + + def probe_ports_list(self): + if isinstance(self.probe_ports, list): + good = True + for p in self.probe_ports: + i = p.find('/') + port = p[:i] + proto = p[(i + 1):] + if proto not in ['udp', 'tcp']: + good = False + break + if proto == 'udp': + raise DMakeException("TODO: udp support in dmake_wait_for_it") + try: + if int(port) < 0: + good = False + break + except: + good = False + break + + if good: + return ','.join(self.probe_ports) + elif self.probe_ports in ['auto', 'none']: + return self.probe_ports + + raise DMakeException("Badly formatted probe ports.") + +class DockerLinkSerializer(YAML2PipelineSerializer, ProbePortMixin): image_name = FieldSerializer("string", example = "mongo:3.2", help_text = "Name and tag of the image to launch.") link_name = FieldSerializer("string", example = "mongo", help_text = "Link name.") volumes = FieldSerializer("array", child = FieldSerializer([SharedVolumeMountSerializer(), VolumeMountSerializer()]), default = [], example = ["datasets:/datasets", "/mnt:/mnt"], help_text = "Either shared volumes to mount. Or: for the 'shell' command only. The list of volumes to mount on the link. It must be in the form ./host/path:/absolute/container/path. Host path is relative to the dmake file.") need_gpu = FieldSerializer("bool", default = False, help_text = "Whether the docker link needs to be run on a GPU node.") # TODO: This field is badly named. Link are used by the run command also, nothing to do with testing or not. It should rather be: 'docker_options' testing_options = FieldSerializer("string", default = "", example = "-v /mnt:/data", help_text = "Additional Docker options when testing on Jenkins.") - probe_ports = FieldSerializer(["string", "array"], default = "auto", child = "string", help_text = "Either 'none', 'auto' or a list of ports in the form 1234/tcp or 1234/udp") env = FieldSerializer("dict", child = "string", default = {}, example = {'REDIS_URL': '${REDIS_URL}'}, help_text = "Additional environment variables defined when running this image.") env_exports = FieldSerializer("dict", child = "string", default = {}, help_text = "A set of environment variables that will be exported in services that use this link when testing.") @@ -437,33 +466,6 @@ def get_env(self, context_env): def get_shared_volumes(self): return [volume.get_shared_volume() for volume in self.volumes if isinstance(volume, SharedVolumeMountSerializer)] - def probe_ports_list(self): - if isinstance(self.probe_ports, list): - good = True - for p in self.probe_ports: - i = p.find('/') - port = p[:i] - proto = p[(i + 1):] - if proto not in ['udp', 'tcp']: - good = False - break - if proto == 'udp': - raise DMakeException("TODO: udp support in dmake_wait_for_it") - try: - if int(port) < 0: - good = False - break - except: - good = False - break - - if good: - return ','.join(self.probe_ports) - elif self.probe_ports in ['auto', 'none']: - return self.probe_ports - - raise DMakeException("Badly formatted probe ports.") - def get_docker_run_gpu_cmd_prefix(self): return get_docker_run_gpu_cmd_prefix(self.need_gpu, 'docker link', self.link_name) @@ -834,7 +836,7 @@ def get_cmd(self): cmd = common.escape_cmd(cmd) return 'bash -c "%s"' % cmd -class DeployConfigSerializer(YAML2PipelineSerializer): +class DeployConfigSerializer(YAML2PipelineSerializer, ProbePortMixin): docker_image = DockerImageFieldSerializer() docker_opts = FieldSerializer("string", default = "", example = "--privileged", help_text = "Docker options to add.") env_override = FieldSerializer("dict", child = "string", optional = True, default = {}, help_text = "Extra environment variables for this service. Overrides dmake.yml root `env`, with variable substitution evaluated from it.", example = {'INFO': '${BRANCH}-${BUILD}'}) @@ -1054,6 +1056,14 @@ def _validate_(self, file, needed_migrations, data, field_name): if common.is_string(data): data = {'service_name': data} result = super(NeededServiceSerializer, self)._validate_(file, needed_migrations=needed_migrations, data=data, field_name=field_name) + + # We link the service by default + if self.link_name is None: + # We link the service using its name, unless the service name has ':' in which case it is a variant + # and it is better to let the user chose the link name if needed + if self.service_name.find(':') < 0: + self.__fields__['link_name'] = self.service_name + if self.link_name and \ not allowed_link_name_pattern.match(self.link_name): raise ValidationError("Invalid link name '%s': only '[a-z0-9-]{1,63}' is allowed. " % (self.link_name)) @@ -1136,7 +1146,7 @@ class DMakeFileSerializer(YAML2PipelineSerializer): env = FieldSerializer(["file", EnvSerializer()], optional = True, help_text = "Environment variables to embed in built docker images.") volumes = FieldSerializer("array", child = SharedVolumeSerializer(), default = [], help_text = "List of shared volumes usabled on services and docker_links", example = ['datasets']) docker = FieldSerializer([FieldSerializer("file", help_text = "to another dmake file (which will be added to dependencies) that declares a docker field, in which case it replaces this file's docker field."), DockerSerializer()], help_text = "The environment in which to build and deploy.") - docker_links = FieldSerializer("array", child = DockerLinkSerializer(), default = [], help_text = "List of link to create, they are shared across the whole application, so potentially across multiple dmake files.") + docker_links = FieldSerializer("array", deprecated="Use 'services' instead and indicate the external image as a string in services:config:docker_image", child=DockerLinkSerializer(), default=[], help_text="List of link to create, they are shared across the whole application, so potentially across multiple dmake files.") build = BuildSerializer(help_text = "Commands to run for building the application.") pre_test_commands = FieldSerializer("array", default = [], child = "string", help_text = "Deprecated, not used anymore, will be removed later. Use `tests.commands` instead.") post_test_commands = FieldSerializer("array", default = [], child = "string", help_text = "Deprecated, not used anymore, will be removed later. Use `tests.commands` instead.") @@ -1332,7 +1342,7 @@ def generate_run(self, commands, service_name, docker_links, service_customizati docker_opts, image_name, env = self._generate_run_docker_opts_(commands, service, docker_links, additional_env_variables = additional_customization_env_variables) docker_opts += service.tests.get_mounts_opt(service_name, self.__path__, env) - docker_cmd = 'dmake_run_docker_daemon "%s" "%s" "%s" "" %s -i %s' % (self.app_name, unique_service_name, link_name or "", docker_opts, image_name) + docker_cmd = 'dmake_run_service "%s" "%s" "%s" "%s" %s -i %s' % (self.app_name, unique_service_name, link_name or "", service.config.probe_ports_list(), docker_opts, image_name) docker_cmd = service.get_docker_run_gpu_cmd_prefix() + docker_cmd # Run daemon diff --git a/dmake/utils/dmake_run_docker_daemon b/dmake/utils/dmake_run_docker_daemon index f634cc9a..06896a4a 100755 --- a/dmake/utils/dmake_run_docker_daemon +++ b/dmake/utils/dmake_run_docker_daemon @@ -1,16 +1,15 @@ #!/bin/bash # # Usage: -# ID=$(dmake_run_docker_daemon APP_NAME SERVICE_NAME LINK_NAME NAME ARGS...) +# ID=$(dmake_run_docker_daemon APP_NAME LINK_NAME NAME ARGS...) # # Result: # Run a docker daemon, save container ID to list of containers to stop and return the container ID -# if SERVICE_NAME is non-empty, save container ID as the daemon for SERVICE_NAME (in the list of daemons ID) # if LINK_NAME is non-empty, save container ID as the link for LINK_NAME (in /links/.id) test "${DMAKE_DEBUG}" = "1" && set -x -if [ $# -lt 2 ]; then +if [ $# -lt 3 ]; then dmake_fail "$0: Missing arguments" echo "exit 1" exit 1 @@ -24,7 +23,6 @@ fi set -e APP_NAME=$1; shift -SERVICE_NAME=$1; shift LINK_NAME=$1; shift NAME=$1; shift @@ -45,10 +43,6 @@ while [ 1 = 1 ]; do sleep 1 done -if [ ! -z "${SERVICE_NAME}" ]; then - echo "${CONTAINER_ID} ${SERVICE_NAME}" >> ${DMAKE_TMP_DIR}/daemon_ids.txt -fi - if [ ! -z "${LINK_NAME}" ]; then CACHE_DIR="${DMAKE_TMP_DIR}/links/${APP_NAME}" ID_FILE="${CACHE_DIR}/${LINK_NAME}.id" diff --git a/dmake/utils/dmake_run_docker_link b/dmake/utils/dmake_run_docker_link index f83792fb..3b724478 100755 --- a/dmake/utils/dmake_run_docker_link +++ b/dmake/utils/dmake_run_docker_link @@ -38,18 +38,5 @@ while [ 1 = 1 ]; do done docker pull ${IMAGE_NAME} 2> /dev/null || : -CONTAINER_ID=$(dmake_run_docker_daemon "${APP_NAME}" "" "${LINK_NAME}" "${CONTAINER_NAME}" ${OPTIONS[@]} ${VOLUMES} -i ${IMAGE_NAME}) - - -if [ "${PROBE_PORTS}" = "none" ]; then - : -else - if [ "${PROBE_PORTS}" = "auto" ]; then - PROBE_PORTS=$(docker ps -f id=${CONTAINER_ID} --format "{{.Ports}}" | sed "s/ *//g" | sed "s/\([0-9]*\.\)\{3\}[0-9]*:[0-9]*->//g") - fi - if [ ! -z "${PROBE_PORTS}" ]; then - ROOT=$(dirname $0) - LINK_OPT="--link ${CONTAINER_ID}:${LINK_NAME}" - docker run --rm ${LINK_OPT} -v ${ROOT}/dmake_wait_for_it:/usr/bin/dmake_wait_for_it -v ${ROOT}/dmake_wait_for_it_wrapper:/usr/bin/dmake_wait_for_it_wrapper -i ubuntu dmake_wait_for_it_wrapper "${LINK_NAME}" "${PROBE_PORTS}" - fi -fi +CONTAINER_ID=$(dmake_run_docker_daemon "${APP_NAME}" "${LINK_NAME}" "${CONTAINER_NAME}" ${OPTIONS[@]} ${VOLUMES} -i ${IMAGE_NAME}) +dmake_wait_for_probe_ports "${CONTAINER_ID}" "${PROBE_PORTS}" diff --git a/dmake/utils/dmake_run_service b/dmake/utils/dmake_run_service new file mode 100755 index 00000000..04af3001 --- /dev/null +++ b/dmake/utils/dmake_run_service @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Usage: +# ID=$(dmake_run_service APP_NAME SERVICE_NAME LINK_NAME PROBE_PORTS ARGS...) +# +# Result: +# Run a docker container associated with a service, save container ID to list of containers to stop and return the container ID +# It also saves the container ID as the daemon for SERVICE_NAME (in the list of daemons ID) +# if LINK_NAME is non-empty, save container ID as the link for LINK_NAME (in /links/.id) + +test "${DMAKE_DEBUG}" = "1" && set -x + +if [ $# -lt 4 ]; then + dmake_fail "$0: Missing arguments" + echo "exit 1" + exit 1 +fi + +if [ -z "${DMAKE_TMP_DIR}" ]; then + dmake_fail "Missing environment variable DMAKE_TMP_DIR" + exit 1 +fi + +set -e + +APP_NAME=$1; shift +SERVICE_NAME=$1; shift +LINK_NAME=$1; shift +PROBE_PORTS=$1; shift + +CONTAINER_ID=$(dmake_run_docker_daemon "${APP_NAME}" "${LINK_NAME}" "" "$@") +dmake_wait_for_probe_ports "${CONTAINER_ID}" "${PROBE_PORTS}" + +if [ ! -z "${SERVICE_NAME}" ]; then + echo "${CONTAINER_ID} ${SERVICE_NAME}" >> ${DMAKE_TMP_DIR}/daemon_ids.txt +fi + +echo ${CONTAINER_ID} diff --git a/dmake/utils/dmake_wait_for_probe_ports b/dmake/utils/dmake_wait_for_probe_ports new file mode 100755 index 00000000..d51fb75c --- /dev/null +++ b/dmake/utils/dmake_wait_for_probe_ports @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Use this script to wait on probe ports +# +# Usage: +# dmake_wait_for_probe_ports CONTAINER_ID PORTS +# +# Result: +# Wait for ports to be open + +test "${DMAKE_DEBUG}" = "1" && set -x + +if [ $# -ne 2 ]; then + >&2 echo -e "$0: Missing arguments" + exit 1 +fi + +set -e + +CONTAINER_ID=${1} +PROBE_PORTS=${2} + +if [ "${PROBE_PORTS}" = "none" ]; then + : +else + if [ "${PROBE_PORTS}" = "auto" ]; then + PROBE_PORTS=$(docker ps -f id=${CONTAINER_ID} --format "{{.Ports}}" | sed "s/ *//g" | sed "s/\([0-9]*\.\)\{3\}[0-9]*:[0-9]*->//g") + fi + if [ ! -z "${PROBE_PORTS}" ]; then + ROOT=$(dirname $0) + >&2 docker run --rm --link ${CONTAINER_ID}:link_to_wait -v ${ROOT}/dmake_wait_for_it:/usr/bin/dmake_wait_for_it -v ${ROOT}/dmake_wait_for_it_wrapper:/usr/bin/dmake_wait_for_it_wrapper -i ubuntu dmake_wait_for_it_wrapper "link_to_wait" "${PROBE_PORTS}" + fi +fi