Faster CI

circle.yml

@@ -7,31 +7,42 @@ jobs:
     docker: [{image: 'docker:17.05-git'}]
     steps:
       - checkout
-      - setup_remote_docker
+      - setup_remote_docker:
+            reusable: true
+            exclusive: false
       - run:
           command: docker version
       - run:
           name: "Lint"
           command: |
             dockerfile=dockerfiles/Dockerfile.lint
             echo "COPY . ." >> $dockerfile
-            docker build -f $dockerfile --tag cli-linter .
-            docker run cli-linter
+            docker build -f $dockerfile --tag cli-linter:$CIRCLE_BUILD_NUM .
+            docker run --rm cli-linter:$CIRCLE_BUILD_NUM
 
   cross:
     working_directory: /work
     docker: [{image: 'docker:17.05-git'}]
+    parallelism: 3
     steps:
       - checkout
-      - setup_remote_docker
+      - setup_remote_docker:
+            reusable: true
+            exclusive: false
       - run:
           name: "Cross"
           command: |
             dockerfile=dockerfiles/Dockerfile.cross
             echo "COPY . ." >> $dockerfile
-            docker build -f $dockerfile --tag cli-builder .
-            docker run --name cross cli-builder make cross
-            docker cp cross:/go/src/github.com/docker/cli/build /work/build
+            docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM .
+            name=cross-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
+            docker run \
+                -e CROSS_GROUP=$CIRCLE_NODE_INDEX \
+                --name $name cli-builder:$CIRCLE_BUILD_NUM \
+                make cross
+            docker cp \
+                $name:/go/src/github.com/docker/cli/build \
+                /work/build
       - store_artifacts:
           path: /work/build
 
@@ -40,36 +51,46 @@ jobs:
     docker: [{image: 'docker:17.05-git'}]
     steps:
       - checkout
-      - setup_remote_docker
+      - setup_remote_docker:
+            reusable: true
+            exclusive: false
       - run:
           name: "Unit Test with Coverage"
           command: |
             dockerfile=dockerfiles/Dockerfile.dev
             echo "COPY . ." >> $dockerfile
-            docker build -f $dockerfile --tag cli-builder .
-            docker run --name test cli-builder make test-coverage
+            docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM .
+            docker run --name \
+                test-$CIRCLE_BUILD_NUM cli-builder:$CIRCLE_BUILD_NUM \
+                make test-coverage
 
       - run:
           name: "Upload to Codecov"
           command: |
-            docker cp test:/go/src/github.com/docker/cli/coverage.txt coverage.txt
+            docker cp \
+                test-$CIRCLE_BUILD_NUM:/go/src/github.com/docker/cli/coverage.txt \
+                coverage.txt
             apk add -U bash curl
             curl -s https://codecov.io/bash | bash
 
   validate:
     working_directory: /work
     docker: [{image: 'docker:17.05-git'}]
     steps:
+      - run: apk add -U git openssh
       - checkout
-      - setup_remote_docker
+      - setup_remote_docker:
+            reusable: true
+            exclusive: false
       - run:
           name: "Validate Vendor, Docs, and Code Generation"
           command: |
             dockerfile=dockerfiles/Dockerfile.dev
             echo "COPY . ." >> $dockerfile
             rm -f .dockerignore # include .git
-            docker build -f $dockerfile --tag cli-builder .
-            docker run cli-builder make -B vendor compose-jsonschema manpages yamldocs
+            docker build -f $dockerfile --tag cli-builder-with-git:$CIRCLE_BUILD_NUM .
+            docker run --rm cli-builder-with-git:$CIRCLE_BUILD_NUM \
+                make -B vendor compose-jsonschema manpages yamldocs
 
 workflows:
   version: 2

dockerfiles/Dockerfile.cross

@@ -1,18 +1,2 @@
-
-FROM    golang:1.8.3
-
-# allow replacing httpredir or deb mirror
-ARG     APT_MIRROR=deb.debian.org
-RUN     sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list
-
-RUN     apt-get update -qq && apt-get install -y -q \
-            libltdl-dev \
-            gcc-mingw-w64 \
-            parallel \
-            ;
-
-COPY    dockerfiles/osx-cross.sh /tmp/
-RUN     /tmp/osx-cross.sh
-ENV     PATH /osxcross/target/bin:$PATH
-
+FROM    dockercore/golang-cross@sha256:d24e7affa3a85d460d2303c2549f03fc866f2b97d771ccf07b0e6e2b411dd207
 WORKDIR /go/src/github.com/docker/cli

scripts/build/cross

@@ -6,13 +6,28 @@
 set -eu -o pipefail
 
 BUILDDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+export SHELL=bash
 
-echo "Building binaries for all platforms"
-SHELL=/bin/bash parallel ::: \
+jobs=(
     "$BUILDDIR/windows" \
     "$BUILDDIR/osx" \
     "GOOS=linux GOARCH=amd64   $BUILDDIR/binary" \
     "GOOS=linux GOARCH=arm     $BUILDDIR/binary" \
     "GOOS=linux GOARCH=ppc64le $BUILDDIR/binary" \
     "GOOS=linux GOARCH=s390x   $BUILDDIR/binary" \
-    ;
+)
+
+# Outside of circleCI run all at once. On circleCI run two at a time because
+# each continer has access to two cores.
+group=${CROSS_GROUP-"all"}
+
+if [ "$group" == "all" ]; then
+
+    echo "Building binaries for all platforms"
+    parallel ::: "${jobs[@]}"
+    exit 0
+
+fi
+
+declare -i start=$group*2
+parallel ::: "${jobs[@]:$start:2}"