From bad2df59f4b949549bc5591c17dee477e83adfad Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Tue, 23 Jul 2019 15:39:57 -0700 Subject: [PATCH 1/7] Bump Go to 1.11.12 go1.11.12 (released 2019/07/08) includes fixes to the compiler and the linker. See the Go 1.11.12 milestone on our issue tracker for details: https://github.com/golang/go/issues?q=milestone%3AGo1.11.12 Full diff: https://github.com/golang/go/compare/go1.11.11...go1.11.12 Signed-off-by: Kir Kolyshkin --- Dockerfile | 2 +- Dockerfile.e2e | 2 +- Dockerfile.simple | 2 +- Dockerfile.windows | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c8a62fb206a76..d65350f557f3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ # the case. Therefore, you don't have to disable it anymore. # -FROM golang:1.11.11 AS base +FROM golang:1.11.12 AS base # allow replacing httpredir or deb mirror ARG APT_MIRROR=deb.debian.org RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list diff --git a/Dockerfile.e2e b/Dockerfile.e2e index 4b6addf4e2a32..3ace0fbd98e6e 100644 --- a/Dockerfile.e2e +++ b/Dockerfile.e2e @@ -1,5 +1,5 @@ ## Step 1: Build tests -FROM golang:1.11.11-alpine3.9 as builder +FROM golang:1.11.12-alpine3.9 as builder RUN apk add --update \ bash \ diff --git a/Dockerfile.simple b/Dockerfile.simple index b188c5d99f922..595eb15cf44e1 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -5,7 +5,7 @@ # This represents the bare minimum required to build and test Docker. -FROM golang:1.11.11-stretch +FROM golang:1.11.12-stretch # allow replacing httpredir or deb mirror ARG APT_MIRROR=deb.debian.org diff --git a/Dockerfile.windows b/Dockerfile.windows index b93d7d5721fab..1c280fa4d65c7 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -161,7 +161,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. # - FROM_DOCKERFILE is used for detection of building within a container. -ENV GO_VERSION=1.11.11 ` +ENV GO_VERSION=1.11.12 ` GIT_VERSION=2.11.1 ` GOPATH=C:\go ` FROM_DOCKERFILE=1 From a4ae2af0e59018a309d4bc71ced729b34e906564 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Tue, 2 Oct 2018 17:57:42 -0700 Subject: [PATCH 2/7] Dockerfile.e2e: simplify apk calls As of Alpine Linux 3.3 (or 3.2?) there exists a new --no-cache option for apk. It allows users to install packages with an index that is updated and used on-the-fly and not cached locally. This avoids the need to use --update and remove /var/cache/apk/* when done installing packages. Signed-off-by: Kir Kolyshkin (cherry picked from commit 3b44dd66a4b316049df86521c491b50466d9b55d) Signed-off-by: Kir Kolyshkin --- Dockerfile.e2e | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Dockerfile.e2e b/Dockerfile.e2e index 3ace0fbd98e6e..620c4e3da8f6c 100644 --- a/Dockerfile.e2e +++ b/Dockerfile.e2e @@ -1,14 +1,13 @@ ## Step 1: Build tests FROM golang:1.11.12-alpine3.9 as builder -RUN apk add --update \ +RUN apk --no-cache add \ bash \ btrfs-progs-dev \ build-base \ curl \ lvm2-dev \ - jq \ - && rm -rf /var/cache/apk/* + jq RUN mkdir -p /go/src/github.com/docker/docker/ WORKDIR /go/src/github.com/docker/docker/ @@ -43,7 +42,7 @@ RUN mkdir -p /output/tests && find . -name test.main -exec cp --parents '{}' /ou FROM alpine:3.9 as runner # GNU tar is used for generating the emptyfs image -RUN apk add --update \ +RUN apk --no-cache add \ bash \ ca-certificates \ g++ \ @@ -51,8 +50,7 @@ RUN apk add --update \ iptables \ pigz \ tar \ - xz \ - && rm -rf /var/cache/apk/* + xz # Add an unprivileged user to be used for tests which need it RUN addgroup docker && adduser -D -G docker unprivilegeduser -s /bin/ash From 6e603e957eec3a46db4d8d69c3a7946c4d2e9c59 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 16 Jul 2019 12:16:56 +0200 Subject: [PATCH 3/7] Dockerfile: Use APT_MIRROR for security.debian.org as well The fastly cdn mirror we're using also mirrors the debian security repository; ``` Welcome to deb.debian.org (fastly instance)! This is deb.debian.org. This service provides mirrors for the following Debian archive repositories: /debian/ /debian-debug/ /debian-ports/ /debian-security/ The server deb.debian.org does not have packages itself, but the name has SRV records in DNS that let apt in stretch and later find places. ``` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit c8f43b5f6f7c83cfb5570f7f013c7efaa430d285) Signed-off-by: Kir Kolyshkin Conflicts: * Dockerfile (GO_VERSION value differs, and CROSS (#39010) is absent) Signed-off-by: Kir Kolyshkin --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d65350f557f3c..99b3bc49b7c0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,9 +25,9 @@ # FROM golang:1.11.12 AS base -# allow replacing httpredir or deb mirror -ARG APT_MIRROR=deb.debian.org -RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list +ARG APT_MIRROR +RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ + && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list FROM base AS criu # Install CRIU for checkpoint/restore support From d9ba337adb0c3880acd34ffe9b0a15717990876b Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 17 Jul 2019 13:59:16 +0200 Subject: [PATCH 4/7] Dockerfile: use GO_VERSION build-arg for overriding Go version This allows overriding the version of Go without making modifications in the source code, which can be useful to test against multiple versions. For example: make GO_VERSION=1.13beta1 shell Signed-off-by: Sebastiaan van Stijn (cherry picked from commit c6281bc4383b7f9eab617fd73601e8594c93365b) Signed-off-by: Kir Kolyshkin Conflicts: * Dockerfile, Dockerfile.e2e, Dockerfile.simple, Dockerfile.windows: (due to Go version difference, missing CROSS etc.) --- Dockerfile | 4 +++- Dockerfile.e2e | 5 +++-- Dockerfile.simple | 4 +++- Dockerfile.windows | 4 +++- Makefile | 2 +- hack/ci/windows.ps1 | 2 +- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 99b3bc49b7c0a..3ad20aaa92df8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,9 @@ # the case. Therefore, you don't have to disable it anymore. # -FROM golang:1.11.12 AS base +ARG GO_VERSION=1.11.12 + +FROM golang:${GO_VERSION} AS base ARG APT_MIRROR RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list diff --git a/Dockerfile.e2e b/Dockerfile.e2e index 620c4e3da8f6c..9a2ec32d1fba7 100644 --- a/Dockerfile.e2e +++ b/Dockerfile.e2e @@ -1,5 +1,6 @@ -## Step 1: Build tests -FROM golang:1.11.12-alpine3.9 as builder +ARG GO_VERSION=1.11.12 + +FROM golang:${GO_VERSION}-alpine3.9 AS builder RUN apk --no-cache add \ bash \ diff --git a/Dockerfile.simple b/Dockerfile.simple index 595eb15cf44e1..5762742533f99 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -5,7 +5,9 @@ # This represents the bare minimum required to build and test Docker. -FROM golang:1.11.12-stretch +ARG GO_VERSION=1.11.12 + +FROM golang:${GO_VERSION}-stretch # allow replacing httpredir or deb mirror ARG APT_MIRROR=deb.debian.org diff --git a/Dockerfile.windows b/Dockerfile.windows index 1c280fa4d65c7..75ad143b06068 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -158,10 +158,12 @@ FROM microsoft/windowsservercore # Use PowerShell as the default shell SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] +ARG GO_VERSION=1.11.12 + # Environment variable notes: # - GO_VERSION must be consistent with 'Dockerfile' used by Linux. # - FROM_DOCKERFILE is used for detection of building within a container. -ENV GO_VERSION=1.11.12 ` +ENV GO_VERSION=${GO_VERSION} ` GIT_VERSION=2.11.1 ` GOPATH=C:\go ` FROM_DOCKERFILE=1 diff --git a/Makefile b/Makefile index e2539683e13a2..f9fed3e086835 100644 --- a/Makefile +++ b/Makefile @@ -137,7 +137,7 @@ dynbinary: build ## build the linux dynbinaries build: DOCKER_BUILDKIT ?= 1 build: bundles $(warning The docker client CLI has moved to github.com/docker/cli. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n}) - DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . + DOCKER_BUILDKIT="${DOCKER_BUILDKIT}" docker build --build-arg=GO_VERSION ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} ${DOCKER_BUILD_OPTS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . bundles: mkdir bundles diff --git a/hack/ci/windows.ps1 b/hack/ci/windows.ps1 index 6d87f3256b30e..3e4c83a93db47 100644 --- a/hack/ci/windows.ps1 +++ b/hack/ci/windows.ps1 @@ -453,7 +453,7 @@ Try { Write-Host -ForegroundColor Cyan "`n`nINFO: Building the image from Dockerfile.windows at $(Get-Date)..." Write-Host $ErrorActionPreference = "SilentlyContinue" - $Duration=$(Measure-Command { docker build -t docker -f Dockerfile.windows . | Out-Host }) + $Duration=$(Measure-Command { docker build --build-arg=GO_VERSION -t docker -f Dockerfile.windows . | Out-Host }) $ErrorActionPreference = "Stop" if (-not($LastExitCode -eq 0)) { Throw "ERROR: Failed to build image from Dockerfile.windows" From 8f1a27c51f93698b5fe17f0fb043f47794e11cca Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 20 Jul 2019 12:46:07 +0200 Subject: [PATCH 5/7] Dockerfile.windows: trim .0 from Go versions This was an oversight when changing the Dockerfile to use a build-arg; the Windows Dockerfile downloads the Go binaries, which never have a trailing `.0`. This patch makes sure that the trailing zero (if any) is removed. Signed-off-by: Sebastiaan van Stijn (cherry picked from commit c5bd6e3dc7680d6c683496f63dafb1f30f87eaa7) Signed-off-by: Kir Kolyshkin --- Dockerfile.windows | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.windows b/Dockerfile.windows index 75ad143b06068..6dc3a859b208b 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -207,7 +207,7 @@ RUN ` Download-File $location C:\gitsetup.zip; ` ` Write-Host INFO: Downloading go...; ` - Download-File $('https://golang.org/dl/go'+$Env:GO_VERSION+'.windows-amd64.zip') C:\go.zip; ` + Download-File $('https://golang.org/dl/go'+$Env:GO_VERSION.TrimEnd('.0')"+'.windows-amd64.zip') C:\go.zip; ` ` Write-Host INFO: Downloading compiler 1 of 3...; ` Download-File https://raw.githubusercontent.com/jhowardmsft/docker-tdmgcc/master/gcc.zip C:\gcc.zip; ` From a2d32c5654a206954fd0d9a5fd08abd8c03d4523 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 20 Jul 2019 12:54:50 +0200 Subject: [PATCH 6/7] Windows: fix Golang version checks for GO_VERSION build-arg This check was used to make sure we don't bump Go versions independently (Linux/Windows). The Dockerfile switched to using a build-arg to allow overriding the Go version, which rendered this check non-functional. It also fails if Linux versions use a specific variant of the image; 08:41:31 ERROR: Failed 'ERROR: Mismatched GO versions between Dockerfile and Dockerfile.windows. Update your PR to ensure that both files are updated and in sync. ${GO_VERSION}-stretch ${GO_VERSION}' at 07/20/2019 08:41:31 08:41:31 At C:\gopath\src\github.com\docker\docker\hack\ci\windows.ps1:448 char:9 08:41:31 + Throw "ERROR: Mismatched GO versions between Dockerfile and D ... 08:41:31 + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This patch fixes the check by looking for the value of `GO_VERSION` instead of looking at the `FROM` line (which is harder to parse). Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 4fa57a8191b1d23c6466725b688519f83c0ac5dd) Signed-off-by: Kir Kolyshkin --- hack/ci/windows.ps1 | 17 +++-------------- hack/make.ps1 | 2 +- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/hack/ci/windows.ps1 b/hack/ci/windows.ps1 index 3e4c83a93db47..8828f73d01f63 100644 --- a/hack/ci/windows.ps1 +++ b/hack/ci/windows.ps1 @@ -426,20 +426,9 @@ Try { Write-Host -ForegroundColor Green "INFO: Location for testing is $env:TEMP" # CI Integrity check - ensure Dockerfile.windows and Dockerfile go versions match - $goVersionDockerfileWindows=$(Get-Content ".\Dockerfile.windows" | Select-String "^ENV GO_VERSION" | Select-object -First 1).ToString().Replace("ENV GO_VERSION=","").Replace("\","").Replace("``","").Trim() - $goVersionDockerfile=$(Get-Content ".\Dockerfile" | Select-String "^ENV GO_VERSION" | Select-object -First 1) - - # As of go 1.11, Dockerfile changed to be in the format like "FROM golang:1.11.0 AS base". - # If a version number ends with .0 (as in 1.11.0, a convention used in golang docker - # image versions), it needs to be removed (i.e. "1.11.0" becomes "1.11"). - if ($null -eq $goVersionDockerfile) { - $goVersionDockerfile=$(Get-Content ".\Dockerfile" | Select-String "^FROM golang:" | Select-object -First 1) - if ($null -ne $goVersionDockerfile) { - $goVersionDockerfile = $goVersionDockerfile.ToString().Split(" ")[1].Split(":")[1] -replace '\.0$','' - } - } else { - $goVersionDockerfile = $goVersionDockerfile.ToString().Split(" ")[2] - } + $goVersionDockerfileWindows=(Select-String -Path ".\Dockerfile.windows" -Pattern "^ARG[\s]+GO_VERSION=(.*)$").Matches.groups[1].Value + $goVersionDockerfile=(Select-String -Path ".\Dockerfile" -Pattern "^ARG[\s]+GO_VERSION=(.*)$").Matches.groups[1].Value + if ($null -eq $goVersionDockerfile) { Throw "ERROR: Failed to extract golang version from Dockerfile" } diff --git a/hack/make.ps1 b/hack/make.ps1 index ee7aeed480296..be60c0816c53f 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -134,7 +134,7 @@ Function Check-InContainer() { # outside of a container where it may be out of date with master. Function Verify-GoVersion() { Try { - $goVersionDockerfile=(Select-String -Path ".\Dockerfile" -Pattern "^FROM golang:").ToString().Split(" ")[1].SubString(7) -replace '\.0$','' + $goVersionDockerfile=(Select-String -Path ".\Dockerfile" -Pattern "^ARG[\s]+GO_VERSION=(.*)$").Matches.groups[1].Value.TrimEnd(".0") $goVersionInstalled=(go version).ToString().Split(" ")[2].SubString(2) } Catch [Exception] { From 3c1bc29b2c372d0f1e37dc8aac33cdc01b623aa0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Sat, 20 Jul 2019 10:32:08 +0200 Subject: [PATCH 7/7] Pin Dockerfile to -stretch variant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Golang base images switch to buster, which causes some breakage in networking and packages that are no longer available; (`btrfs-tools` is now an empty package, and `libprotobuf-c0-dev` is gone). Some of out tests also start faiilng on stretch, and will have to be investigated further; ``` 15:13:06 --- FAIL: TestRenameAnonymousContainer (3.37s) 15:13:06 rename_test.go:168: assertion failed: 0 (int) != 1 (inspect.State.ExitCode int): container a7fe866d588d65f353f42ffc5ea5288e52700384e1d90850e9c3d4dce8657666 exited with the wrong exitcode: 15:13:38 --- FAIL: TestHostnameDnsResolution (2.23s) 15:13:38 run_linux_test.go:128: assertion failed: 15:13:38 --- ← 15:13:38 +++ → 15:13:38 @@ -1 +1,2 @@ 15:13:38 +ping: bad address 'foobar' 15:13:38 15:13:38 15:13:38 run_linux_test.go:129: assertion failed: 0 (int) != 1 (res.ExitCode int) ``` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ed672bb523cb255d0b2b79837d9c45a7c3255000) Signed-off-by: Kir Kolyshkin --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3ad20aaa92df8..121f1ed2a9d59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ ARG GO_VERSION=1.11.12 -FROM golang:${GO_VERSION} AS base +FROM golang:${GO_VERSION}-stretch AS base ARG APT_MIRROR RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \ && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list