From 2485a1d6f731a0f22bf06ec8485dd69ffd91536f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 17 May 2017 15:57:28 -0400 Subject: [PATCH 1/4] Replace docker.Makefile with a bash script. Signed-off-by: Daniel Nephin --- README.md | 11 ++--- docker.Makefile | 82 ------------------------------------- dockerfiles/Dockerfile.lint | 5 +-- scripts/build/.variables | 3 +- tasks | 53 ++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 91 deletions(-) delete mode 100644 docker.Makefile create mode 100755 tasks diff --git a/README.md b/README.md index 340556d8c77d..7f64c42c3d9e 100644 --- a/README.md +++ b/README.md @@ -9,24 +9,25 @@ Docker EE products. Development =========== -`docker/cli` is developed using Docker. +`docker/cli` is developed using Docker. The `./tasks` script is used to run +build Docker images and run Docker containers. Build a linux binary: ``` -$ make -f docker.Makefile binary +$ ./tasks binary ``` Build binaries for all supported platforms: ``` -$ make -f docker.Makefile cross +$ ./tasks cross ``` Run all linting: ``` -$ make -f docker.Makefile lint +$ ./tasks lint ``` ### In-container development environment @@ -34,7 +35,7 @@ $ make -f docker.Makefile lint Start an interactive development environment: ``` -$ make -f docker.Makefile shell +$ ./tasks shell ``` In the development environment you can run many tasks, including build binaries: diff --git a/docker.Makefile b/docker.Makefile deleted file mode 100644 index 2d7474b583f7..000000000000 --- a/docker.Makefile +++ /dev/null @@ -1,82 +0,0 @@ -# -# github.com/docker/cli -# -# Makefile for developing using Docker -# - -DEV_DOCKER_IMAGE_NAME = docker-cli-dev -LINTER_IMAGE_NAME = docker-cli-lint -CROSS_IMAGE_NAME = docker-cli-cross -MOUNTS = -v "$(CURDIR)":/go/src/github.com/docker/cli -VERSION = $(shell cat VERSION) -ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT - -# build docker image (dockerfiles/Dockerfile.build) -.PHONY: build_docker_image -build_docker_image: - docker build ${DOCKER_BUILD_ARGS} -t $(DEV_DOCKER_IMAGE_NAME) -f ./dockerfiles/Dockerfile.dev . - -# build docker image having the linting tools (dockerfiles/Dockerfile.lint) -.PHONY: build_linter_image -build_linter_image: - docker build ${DOCKER_BUILD_ARGS} -t $(LINTER_IMAGE_NAME) -f ./dockerfiles/Dockerfile.lint . - -.PHONY: build_cross_image -build_cross_image: - docker build ${DOCKER_BUILD_ARGS} -t $(CROSS_IMAGE_NAME) -f ./dockerfiles/Dockerfile.cross . - - -# build executable using a container -binary: build_docker_image - docker run --rm $(ENVVARS) $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make binary - -build: binary - -# clean build artifacts using a container -.PHONY: clean -clean: build_docker_image - docker run --rm $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make clean - -# run go test -.PHONY: test -test: build_docker_image - docker run --rm $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make test - -# build the CLI for multiple architectures using a container -.PHONY: cross -cross: build_cross_image - docker run --rm $(ENVVARS) $(MOUNTS) $(CROSS_IMAGE_NAME) make cross - -.PHONY: watch -watch: build_docker_image - docker run --rm $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make watch - -# start container in interactive mode for in-container development -.PHONY: dev -dev: build_docker_image - docker run -ti $(MOUNTS) -v /var/run/docker.sock:/var/run/docker.sock $(DEV_DOCKER_IMAGE_NAME) ash - -shell: dev - -# run linters in a container -.PHONY: lint -lint: build_linter_image - docker run -ti $(MOUNTS) $(LINTER_IMAGE_NAME) - -# download dependencies (vendor/) listed in vendor.conf, using a container -.PHONY: vendor -vendor: build_docker_image vendor.conf - docker run -ti --rm $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make vendor - -dynbinary: build_cross_image - docker run -ti --rm $(ENVVARS) $(MOUNTS) $(CROSS_IMAGE_NAME) make dynbinary - -## generate man pages from go source and markdown -.PHONY: manpages -manpages: build_docker_image - docker run -ti --rm $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make manpages - -## Generate documentation YAML files consumed by docs repo -.PHONY: yamldocs -yamldocs: build_docker_image - docker run -ti --rm $(MOUNTS) $(DEV_DOCKER_IMAGE_NAME) make yamldocs diff --git a/dockerfiles/Dockerfile.lint b/dockerfiles/Dockerfile.lint index 0b9385e0d14f..aa160e945dbd 100644 --- a/dockerfiles/Dockerfile.lint +++ b/dockerfiles/Dockerfile.lint @@ -1,6 +1,6 @@ FROM golang:1.8.3-alpine -RUN apk add -U git +RUN apk add -U git make ARG GOMETALINTER_SHA=4306381615a2ba2a207f8fcea02c08c6b2b0803f RUN go get github.com/alecthomas/gometalinter && \ @@ -12,5 +12,4 @@ RUN go get github.com/alecthomas/gometalinter && \ WORKDIR /go/src/github.com/docker/cli ENV CGO_ENABLED=0 -ENTRYPOINT ["/usr/local/bin/gometalinter"] -CMD ["--config=gometalinter.json", "./..."] +CMD ["/usr/local/bin/gometalinter", "--config=gometalinter.json", "./..."] diff --git a/scripts/build/.variables b/scripts/build/.variables index eb87c382681b..6c819bd7cc2b 100755 --- a/scripts/build/.variables +++ b/scripts/build/.variables @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -eu -VERSION=${VERSION:-"unknown-version"} +default_version=$(cat VERSION) +VERSION=${VERSION:-"$default_version"} GITCOMMIT=${GITCOMMIT:-$(git rev-parse --short HEAD 2> /dev/null || true)} BUILDTIME=${BUILDTIME:-$(date --utc --rfc-3339 ns 2> /dev/null | sed -e 's/ /T/')} diff --git a/tasks b/tasks new file mode 100755 index 000000000000..731d2c84fb5a --- /dev/null +++ b/tasks @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +dev_image=docker-cli-dev +linter_image=docker-cli-lint +cross_image=docker-cli-cross +mounts="-v $PWD:/go/src/github.com/docker/cli" + +if [ -t 1 ] ; then use_tty="-ti"; else use_tty=""; fi + +function run_task { + local task=$1 + + if [[ "$task" = *shell ]]; then + local cmd=sh + else + local cmd="make $task" + fi + + case $task in + lint|lint-shell) + docker build -t "$linter_image" -f ./dockerfiles/Dockerfile.lint . + docker run --rm $use_tty $mounts "$linter_image" $cmd + ;; + cross|dynbinary|cross-shell) + docker build -t "$cross_image" -f ./dockerfiles/Dockerfile.cross . + docker run --rm $use_tty $mounts "$cross_image" $cmd + ;; + *) + docker build -t "$dev_image" -f ./dockerfiles/Dockerfile.dev . + docker run --rm $use_tty $mounts "$dev_image" $cmd + ;; + esac +} + +function usage { + cat < Date: Fri, 19 May 2017 15:42:40 -0400 Subject: [PATCH 2/4] Support NO_BINDMOUNT in tasks Use ./tasks in circleci List task targets with './tasks --help' Signed-off-by: Daniel Nephin --- .gitignore | 1 + README.md | 16 ++++------ circle.yml | 46 ++++++++-------------------- scripts/build/.variables | 3 +- tasks | 65 +++++++++++++++++++++++++++++----------- 5 files changed, 68 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index fa99439ac866..e25355c9b5a7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ cli/winresources/rsrc_amd64.syso /docs/yaml/gen/ coverage.txt profile.out +.cid diff --git a/README.md b/README.md index 7f64c42c3d9e..10317b35a764 100644 --- a/README.md +++ b/README.md @@ -9,27 +9,20 @@ Docker EE products. Development =========== -`docker/cli` is developed using Docker. The `./tasks` script is used to run -build Docker images and run Docker containers. +The `./tasks` script allows you to build and develop the cli with Docker. Build a linux binary: - ``` $ ./tasks binary ``` -Build binaries for all supported platforms: - -``` -$ ./tasks cross -``` - Run all linting: - ``` $ ./tasks lint ``` +You can see a full list of tasks with `./tasks --help`. + ### In-container development environment Start an interactive development environment: @@ -38,7 +31,8 @@ Start an interactive development environment: $ ./tasks shell ``` -In the development environment you can run many tasks, including build binaries: +From the interactive development shell you can run tasks defined in the +Makefile. For example, to build a binary you would run: ``` $ make binary diff --git a/circle.yml b/circle.yml index 4552ee6dd941..4a6f342e2b94 100644 --- a/circle.yml +++ b/circle.yml @@ -4,7 +4,7 @@ jobs: lint: working_directory: /work - docker: [{image: 'docker:17.05-git'}] + docker: [{image: 'dnephin/docker-circleci-task-runner:17.05'}] steps: - checkout - setup_remote_docker: @@ -15,14 +15,11 @@ jobs: - run: name: "Lint" command: | - dockerfile=dockerfiles/Dockerfile.lint - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-linter:$CIRCLE_BUILD_NUM . - docker run --rm cli-linter:$CIRCLE_BUILD_NUM + TASK_UNIQUE_ID=$CIRCLE_BUILD_NUM ./tasks lint cross: working_directory: /work - docker: [{image: 'docker:17.05-git'}] + docker: [{image: 'dnephin/docker-circleci-task-runner:17.05'}] parallelism: 3 steps: - checkout @@ -32,23 +29,15 @@ jobs: - run: name: "Cross" command: | - dockerfile=dockerfiles/Dockerfile.cross - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM . - name=cross-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX - docker run \ - -e CROSS_GROUP=$CIRCLE_NODE_INDEX \ - --name $name cli-builder:$CIRCLE_BUILD_NUM \ - make cross - docker cp \ - $name:/go/src/github.com/docker/cli/build \ - /work/build + TASK_UNIQUE_ID=$CIRCLE_BUILD_NUM CID_FILENAME=.cid ./tasks cross + docker cp $(cat .cid):/go/src/github.com/docker/cli/build /work/build + - store_artifacts: path: /work/build test: working_directory: /work - docker: [{image: 'docker:17.05-git'}] + docker: [{image: 'dnephin/docker-circleci-task-runner:17.05'}] steps: - checkout - setup_remote_docker: @@ -57,25 +46,17 @@ jobs: - run: name: "Unit Test with Coverage" command: | - dockerfile=dockerfiles/Dockerfile.dev - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM . - docker run --name \ - test-$CIRCLE_BUILD_NUM cli-builder:$CIRCLE_BUILD_NUM \ - make test-coverage + TASK_UNIQUE_ID=$CIRCLE_BUILD_NUM CID_FILENAME=.cid ./tasks test-coverage - run: name: "Upload to Codecov" command: | - docker cp \ - test-$CIRCLE_BUILD_NUM:/go/src/github.com/docker/cli/coverage.txt \ - coverage.txt - apk add -U bash curl + docker cp $(cat .cid):/go/src/github.com/docker/cli/coverage.txt coverage.txt curl -s https://codecov.io/bash | bash validate: working_directory: /work - docker: [{image: 'docker:17.05-git'}] + docker: [{image: 'dnephin/docker-circleci-task-runner:17.05'}] steps: - run: apk add -U git openssh - checkout @@ -85,12 +66,9 @@ jobs: - run: name: "Validate Vendor, Docs, and Code Generation" command: | - dockerfile=dockerfiles/Dockerfile.dev - echo "COPY . ." >> $dockerfile rm -f .dockerignore # include .git - docker build -f $dockerfile --tag cli-builder-with-git:$CIRCLE_BUILD_NUM . - docker run --rm cli-builder-with-git:$CIRCLE_BUILD_NUM \ - make -B vendor compose-jsonschema manpages yamldocs + export TASK_UNIQUE_ID=$CIRCLE_BUILD_NUM + ./tasks vendor compose-jsonschema manpages yamldocs workflows: version: 2 diff --git a/scripts/build/.variables b/scripts/build/.variables index 6c819bd7cc2b..d51115d90117 100755 --- a/scripts/build/.variables +++ b/scripts/build/.variables @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -eu -default_version=$(cat VERSION) +scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +default_version=$(cat $scriptdir/../../VERSION) VERSION=${VERSION:-"$default_version"} GITCOMMIT=${GITCOMMIT:-$(git rev-parse --short HEAD 2> /dev/null || true)} BUILDTIME=${BUILDTIME:-$(date --utc --rfc-3339 ns 2> /dev/null | sed -e 's/ /T/')} diff --git a/tasks b/tasks index 731d2c84fb5a..c74900851157 100755 --- a/tasks +++ b/tasks @@ -2,12 +2,10 @@ set -eu -o pipefail -dev_image=docker-cli-dev -linter_image=docker-cli-lint -cross_image=docker-cli-cross -mounts="-v $PWD:/go/src/github.com/docker/cli" - -if [ -t 1 ] ; then use_tty="-ti"; else use_tty=""; fi +unique_id=${TASK_UNIQUE_ID:-$USER} +dev_image="docker-cli-dev:$unique_id" +linter_image="docker-cli-lint:$unique_id" +cross_image="docker-cli-cross:$unique_id" function run_task { local task=$1 @@ -17,37 +15,70 @@ function run_task { else local cmd="make $task" fi - + case $task in lint|lint-shell) - docker build -t "$linter_image" -f ./dockerfiles/Dockerfile.lint . - docker run --rm $use_tty $mounts "$linter_image" $cmd + docker_build_and_run "$linter_image" ./dockerfiles/Dockerfile.lint "$cmd" ;; cross|dynbinary|cross-shell) - docker build -t "$cross_image" -f ./dockerfiles/Dockerfile.cross . - docker run --rm $use_tty $mounts "$cross_image" $cmd + docker_build_and_run "$cross_image" ./dockerfiles/Dockerfile.cross "$cmd" ;; *) - docker build -t "$dev_image" -f ./dockerfiles/Dockerfile.dev . - docker run --rm $use_tty $mounts "$dev_image" $cmd + docker_build_and_run "$dev_image" ./dockerfiles/Dockerfile.dev "$cmd" ;; esac } +function docker_build_and_run { + local image=$1 + local dockerfile=$2 + local cmd=$3 + local dockerfile_source= + local cidfile= + local remove="--rm" + local envvars="-e VERSION -e GITCOMMIT -e BUILDTIME -e LDFLAGS" + # Use an array to preserve whitespace in $PWD + local mounts=(-v "$PWD:/go/src/github.com/docker/cli") + if [ -t 1 ] ; then local use_tty="-ti"; else local use_tty=""; fi + + if [ -n "${DOCKER_HOST:-}" ] || [ -n "${NO_BINDMOUNT:-}" ]; then + dockerfile_source="$(cat $dockerfile <(echo COPY . .))" + mounts=() + dockerfile="-" + fi + + if [ -n "${CID_FILENAME:-}" ]; then + cidfile="--cidfile $CID_FILENAME" + remove= + fi + + echo "$dockerfile_source" | docker build -t "$image" -f "$dockerfile" . + docker run $remove $envvars $cidfile $use_tty ${mounts[@]+"${mounts[@]}"} "$image" $cmd +} + function usage { + local tasks="See Makefile for a list of task names." + if command -v make awk sort grep > /dev/null; then + # this ugly command is supposed to list the targets in a Makefile + tasks="$(set +o pipefail; echo; make -qp | \ + awk -F':' '/^[a-zA-Z0-9][^$#\/\t=]*:([^=]|$)/ {split($1,A,/ /); for(i in A) print " " A[i]}' | \ + sort -u | grep -v Makefile)" + tasks="TASK may be one of: $tasks" + fi + cat < Date: Fri, 23 Jun 2017 00:52:34 -0400 Subject: [PATCH 3/4] Add a warning when make is run outside of a container. The warning can be disabled by setting the environment variable Signed-off-by: Daniel Nephin --- Makefile | 2 ++ circle.yml | 1 - scripts/warn-outside-container | 15 +++++++++++++++ tasks | 19 ++++++++++++++++--- 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100755 scripts/warn-outside-container diff --git a/Makefile b/Makefile index dbb3d3bfefc8..4c836cb8dd30 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ # all: binary +_:=$(shell ./scripts/warn-outside-container $(MAKECMDGOALS)) + # remove build artifacts .PHONY: clean clean: diff --git a/circle.yml b/circle.yml index 4a6f342e2b94..435982534d28 100644 --- a/circle.yml +++ b/circle.yml @@ -58,7 +58,6 @@ jobs: working_directory: /work docker: [{image: 'dnephin/docker-circleci-task-runner:17.05'}] steps: - - run: apk add -U git openssh - checkout - setup_remote_docker: reusable: true diff --git a/scripts/warn-outside-container b/scripts/warn-outside-container new file mode 100755 index 000000000000..6592e5a4a277 --- /dev/null +++ b/scripts/warn-outside-container @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -eu + +target="${1:-}" + +if [[ -z "${DISABLE_WARN_OUTSIDE_CONTAINER:-}" ]]; then + ( + echo + echo + echo "WARNING: you are not in a container. Use \"./tasks $target\" or set" + echo "DISABLE_WARN_OUTSIDE_CONTAINER=1 to disable this warning." + echo + echo + ) >&2 +fi diff --git a/tasks b/tasks index c74900851157..a38a7802dd85 100755 --- a/tasks +++ b/tasks @@ -1,5 +1,7 @@ #!/usr/bin/env bash - +# +# Run a development task +# set -eu -o pipefail unique_id=${TASK_UNIQUE_ID:-$USER} @@ -36,7 +38,11 @@ function docker_build_and_run { local dockerfile_source= local cidfile= local remove="--rm" - local envvars="-e VERSION -e GITCOMMIT -e BUILDTIME -e LDFLAGS" + local envvars="-e VERSION \ + -e GITCOMMIT \ + -e BUILDTIME \ + -e LDFLAGS \ + -e DISABLE_WARN_OUTSIDE_CONTAINER=1" # Use an array to preserve whitespace in $PWD local mounts=(-v "$PWD:/go/src/github.com/docker/cli") if [ -t 1 ] ; then local use_tty="-ti"; else local use_tty=""; fi @@ -53,7 +59,14 @@ function docker_build_and_run { fi echo "$dockerfile_source" | docker build -t "$image" -f "$dockerfile" . - docker run $remove $envvars $cidfile $use_tty ${mounts[@]+"${mounts[@]}"} "$image" $cmd + docker run \ + $remove \ + $envvars \ + $cidfile \ + $use_tty \ + ${mounts[@]+"${mounts[@]}"} \ + "$image" \ + $cmd } function usage { From 37c121037f8f3a7f9bf4d46d5add11d9a0f47df7 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 14 Jul 2017 11:06:30 -0400 Subject: [PATCH 4/4] Add TASK_BUILD_ARGS Signed-off-by: Daniel Nephin --- tasks | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tasks b/tasks index a38a7802dd85..7d7cd0cfc27a 100755 --- a/tasks +++ b/tasks @@ -9,6 +9,8 @@ dev_image="docker-cli-dev:$unique_id" linter_image="docker-cli-lint:$unique_id" cross_image="docker-cli-cross:$unique_id" +TASK_BUILD_ARGS=${TASK_BUILD_ARGS:-} + function run_task { local task=$1 @@ -58,7 +60,8 @@ function docker_build_and_run { remove= fi - echo "$dockerfile_source" | docker build -t "$image" -f "$dockerfile" . + echo "$dockerfile_source" | \ + docker build $TASK_BUILD_ARGS -t "$image" -f "$dockerfile" . docker run \ $remove \ $envvars \