From 3469a25b51f4a064505891d027215caf1c0a61eb Mon Sep 17 00:00:00 2001 From: Ying Li Date: Wed, 7 Jun 2017 19:04:58 -0700 Subject: [PATCH 1/9] Bump go verison up to 1.8.3 for security updates, etc., and remove installing the extra mockgen package when installing dependencies. Signed-off-by: Ying Li (cherry picked from commit b5c4335827eade52ab960f037a38085bdb3fcd00) Signed-off-by: Sebastiaan van Stijn --- Makefile | 7 +++---- circle.yml | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 72509345e6..112cd3fd4e 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,6 @@ setup: ## install dependencies # TODO(stevvooe): Install these from the vendor directory @go get -u github.com/golang/lint/golint #@go get -u github.com/kisielk/errcheck - @go get -u github.com/golang/mock/mockgen @go get -u github.com/gordonklaus/ineffassign @go get -u github.com/client9/misspell/cmd/misspell @go get -u github.com/lk4d4/vndr @@ -81,15 +80,15 @@ fmt: ## run go fmt lint: ## run go lint @echo "🐳 $@" - @test -z "$$(golint ./... | grep -v vendor/ | grep -v ".pb.go:" | grep -v ".mock.go" | tee /dev/stderr)" + @test -z "$$(golint ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)" ineffassign: ## run ineffassign @echo "🐳 $@" - @test -z "$$(ineffassign . | grep -v vendor/ | grep -v ".pb.go:" | grep -v ".mock.go" | tee /dev/stderr)" + @test -z "$$(ineffassign . | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)" #errcheck: ## run go errcheck # @echo "🐳 $@" -# @test -z "$$(errcheck ./... | grep -v vendor/ | grep -v ".pb.go:" | grep -v ".mock.go" | tee /dev/stderr)" +# @test -z "$$(errcheck ./... | grep -v vendor/ | grep -v ".pb.go:" | tee /dev/stderr)" build: ## build the go packages @echo "🐳 $@" diff --git a/circle.yml b/circle.yml index 0298bf0369..556d2b4aac 100644 --- a/circle.yml +++ b/circle.yml @@ -5,7 +5,7 @@ machine: PROTOC: "https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip" - GOVERSION: "1.8.1" + GOVERSION: "1.8.3" GOPATH: "$HOME/.go_workspace" WORKDIR: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME" From af5c026f9e6686f04558bdab0b8a461c77e01c2a Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Tue, 13 Jun 2017 17:30:23 -0700 Subject: [PATCH 2/9] Run CI with temporary files on a ramdisk This sets up a tmpfs mount to be used for temporary files created by tests. If this works, hopefully it will fix the excessive I/O latencies that are blocking tests for long periods and causing them to fail. As a side effect, this also stores temporary compilation artifacts on tmpfs. Signed-off-by: Aaron Lehmann (cherry picked from commit ff0a39725b9469db101e01ae0ab4d61a70d1c754) Signed-off-by: Sebastiaan van Stijn --- circle.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 556d2b4aac..8142691639 100644 --- a/circle.yml +++ b/circle.yml @@ -59,7 +59,10 @@ test: # - `checkprotos`: Build and check the proto files *only* once all # tests pass with checked-in generated code. # - `coverage`: Generate coverage reports. - - cd "$WORKDIR" && make ci + - sudo mkdir /tmpfs + - sudo mount -t tmpfs tmpfs /tmpfs + - sudo chown 1000:1000 /tmpfs + - cd "$WORKDIR" && TMPDIR=/tmpfs make ci post: # Report to codecov.io From e05889f0cfb04373739be14152374b4b0dc9e213 Mon Sep 17 00:00:00 2001 From: He Xiaoxi Date: Wed, 9 Aug 2017 14:49:32 +0800 Subject: [PATCH 3/9] Copy the protoc include files to as well when make ci Signed-off-by: He Xiaoxi (cherry picked from commit 1540992284f0b570cd1b24080d8d94239f1765f2) Signed-off-by: Sebastiaan van Stijn --- circle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/circle.yml b/circle.yml index 8142691639..2f82302793 100644 --- a/circle.yml +++ b/circle.yml @@ -33,6 +33,8 @@ dependencies: # Install protoc - wget "$PROTOC" - unzip -o "$(basename $PROTOC)" -d "$HOME" + - sudo cp -R "$HOME/include/google" /usr/local/include + - sudo chmod 777 -R /usr/local/include/google # Setup the GOPATH - mkdir -p "$(dirname $WORKDIR)" From 6ea7c2a659114ab996cc7d65ef2d479bc2ac5c25 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 4 Oct 2017 23:42:49 +0200 Subject: [PATCH 4/9] Bump Go to 1.8.4 Bumps the Go version used to 1.8.4, which contains security fixes; https://groups.google.com/forum/#!topic/golang-announce/1hZYiemnkdE Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 6d2c849853d5f7439b86d579a2ef2e727f6e78de) Signed-off-by: Sebastiaan van Stijn --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 2f82302793..8799c4c7a2 100644 --- a/circle.yml +++ b/circle.yml @@ -5,7 +5,7 @@ machine: PROTOC: "https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip" - GOVERSION: "1.8.3" + GOVERSION: "1.8.4" GOPATH: "$HOME/.go_workspace" WORKDIR: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME" From 4c00ddd511e5e6ece4cc4fca9ca8c6c1a3440114 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 24 Jan 2018 00:34:33 -0800 Subject: [PATCH 5/9] Bump Go to 1.9.3 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 2d27a21b5776827f0918d10db5f62979b7cf2a2b) Signed-off-by: Sebastiaan van Stijn --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 8799c4c7a2..8c4fb7a2f4 100644 --- a/circle.yml +++ b/circle.yml @@ -5,7 +5,7 @@ machine: PROTOC: "https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip" - GOVERSION: "1.8.4" + GOVERSION: "1.9.3" GOPATH: "$HOME/.go_workspace" WORKDIR: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME" From 265a270ad82cd129880e58609651f34480e5e122 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 18 May 2018 21:40:57 +0200 Subject: [PATCH 6/9] Bump Go to 1.10.3 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 0ad63eaff3733cf1175d667c5e2ce6e8660089ba) Signed-off-by: Sebastiaan van Stijn --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 8c4fb7a2f4..9a221290f0 100644 --- a/circle.yml +++ b/circle.yml @@ -5,7 +5,7 @@ machine: PROTOC: "https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip" - GOVERSION: "1.9.3" + GOVERSION: "1.10.3" GOPATH: "$HOME/.go_workspace" WORKDIR: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME" From 8ea7e0719bd455c724b6c5fd87968e6db34e8dc8 Mon Sep 17 00:00:00 2001 From: Ying Li Date: Wed, 6 Jun 2018 14:04:59 -0700 Subject: [PATCH 7/9] Convert the circle config file to v2 format, as circle v1 is deprecated and will be removed in August 2018. Signed-off-by: Ying Li (cherry picked from commit 19114b466a3bf1a538e40d641f809d9d3bf74515) Signed-off-by: Sebastiaan van Stijn --- .circleci/config.yml | 100 +++++++++++++++++++++++++++++++++++++++++++ circle.yml | 71 ------------------------------ 2 files changed, 100 insertions(+), 71 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..eb83412c5b --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,100 @@ +version: 2 +jobs: + build: + working_directory: ~/go/src/github.com/docker/swarmkit + environment: + # Needed to install go + OS: linux + ARCH: amd64 + GOVERSION: 1.10.3 + # Needed to install protoc + PROTOC: https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip + + # Note(cyli): We create a tmpfs mount to be used for temporary files created by tests + # to mitigate the excessive I/O latencies that sometimes cause the tests to fail. + # See https://github.com/docker/swarmkit/pull/2254. + + # There is no way to mount tmpfs volumes in the docker executor, so we are using + # the machine executor. However, this incur a performance overhead + # (https://discuss.circleci.com/t/using-docker-compose-in-2-0/9492/4) + # and in the future could incur additional pricing changes + # (https://circleci.com/docs/2.0/executor-types/#using-machine). + + # One possible hack is the following: + + # /dev/shm in the container is tmpfs although files in /dev/shm are not executable. + # If we specify TMPDIR=/dev/shm, /dev/shm will be used by our tests, which call + # ioutil.TempDir/ioutil.TempFile, to write temporary files. + # We can also specify GOTMPDIR=/tmp or some other non-tmpfs directory + # (see https://golang.org/doc/go1.10#goroot) - this is the directory in which the + # go tool itself will put temporarily compiled test executables, etc. + + # However, using this hack still resulted in occasional WAL test failures, + # so it seems like it does not work, or there may be some other failure. + # It may be something to explore again if the penalty for using the machine + # executor becomes unacceptable. + + machine: true + + steps: + - checkout + + # This would not be needed if we used a golang docker image + - run: + name: Install go + command: | + sudo rm -rf /usr/local/go + rm -rf "$GOPATH" + curl -fsSL -o "$HOME/go.tar.gz" "https://storage.googleapis.com/golang/go$GOVERSION.$OS-$ARCH.tar.gz" + sudo tar -C /usr/local -xzf "$HOME/go.tar.gz" + + - run: + name: Output debugging information + command: | + go version + env + + - run: + name: Install protoc + command: | + curl -fsSL -o "$HOME/$(basename $PROTOC)" "$PROTOC" + unzip -o "$HOME/$(basename $PROTOC)" -d "$HOME" + sudo cp -R "$HOME/include/google" /usr/local/include + sudo chmod 777 -R /usr/local/include/google + sudo cp -R "$HOME/bin/protoc" /usr/local/bin + sudo chmod 777 /usr/local/bin/protoc + + - run: + name: Install test/lint dependencies + command: make setup + + - run: + name: Validate dependency vendoring + command: | + git fetch origin + if test -n "`git diff --stat=1000 origin/master | grep -E \"^[[:space:]]*vendor\"`"; then + make dep-validate; + fi + + # The GOPATH setting would not be needed if we used the golang docker image + - run: + name: Compile/lint/vet/protobuf validation + command: GOPATH="$HOME/go" make check binaries checkprotos + + - run: + name: Run unit tests + command: | + sudo mkdir /tmpfs + sudo mount -t tmpfs tmpfs /tmpfs + sudo chown 1000:1000 /tmpfs + GOPATH="$HOME/go" TMPDIR=/tmpfs make coverage + + - run: + name: Run integration tests + command: | + # TMPFS has already been set up previously in the unit test step + GOPATH="$HOME/go" TMPDIR=/tmpfs make coverage-integration + + - run: + name: Push coverage info to codecov.io + command: bash <(curl -fsSL https://codecov.io/bash) diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 9a221290f0..0000000000 --- a/circle.yml +++ /dev/null @@ -1,71 +0,0 @@ -machine: - environment: - OS: "linux" - ARCH: "amd64" - - PROTOC: "https://github.com/google/protobuf/releases/download/v3.2.0/protoc-3.2.0-linux-x86_64.zip" - - GOVERSION: "1.10.3" - GOPATH: "$HOME/.go_workspace" - - WORKDIR: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME" - -# Disable default dependency management. -# This prevents go dependencies to be automatically installed in order to -# ensure we are exclusively relying on vendored deps. -# -# https://discuss.circleci.com/t/overriding-go-inference-in-the-dependencies-phase/660 -# https://robots.thoughtbot.com/configure-circleci-for-go -dependencies: - pre: - # Wipe out the default go install. - - sudo rm -rf /usr/local/go - - # Force the wipe out of GOPATH to make sure we're not relying on - # external dependencies. - - rm -rf "$GOPATH" - - override: - # Install Go - - wget "https://storage.googleapis.com/golang/go$GOVERSION.$OS-$ARCH.tar.gz" - - sudo tar -C /usr/local -xzf "go$GOVERSION.$OS-$ARCH.tar.gz" - - # Install protoc - - wget "$PROTOC" - - unzip -o "$(basename $PROTOC)" -d "$HOME" - - sudo cp -R "$HOME/include/google" /usr/local/include - - sudo chmod 777 -R /usr/local/include/google - - # Setup the GOPATH - - mkdir -p "$(dirname $WORKDIR)" - - cp -R "$HOME/$CIRCLE_PROJECT_REPONAME" "$WORKDIR" - - # Install dependencies - - cd "$WORKDIR" && make setup - - post: - # Display debugging information in CI logs - - go version - - env - -test: - pre: - # Ensure validation of dependencies - - cd "$WORKDIR" && git fetch origin - - cd "$WORKDIR" && if test -n "`git diff --stat=1000 origin/master | grep -E \"^[[:space:]]*vendor\"`"; then make dep-validate; fi - - override: - # Run all tests in a single pass to ensure we don't move to the next - # step in case of failure. - # - `all`: Format, build and tests. - # - `checkprotos`: Build and check the proto files *only* once all - # tests pass with checked-in generated code. - # - `coverage`: Generate coverage reports. - - sudo mkdir /tmpfs - - sudo mount -t tmpfs tmpfs /tmpfs - - sudo chown 1000:1000 /tmpfs - - cd "$WORKDIR" && TMPDIR=/tmpfs make ci - - post: - # Report to codecov.io - - cd "$WORKDIR" && bash <(curl -s https://codecov.io/bash) From fa73f3102cb22863e99df57b0530bc64f61d11d6 Mon Sep 17 00:00:00 2001 From: Ying Li Date: Thu, 5 Jul 2018 23:36:41 -0700 Subject: [PATCH 8/9] Set the GOPATH for the whole job Signed-off-by: Ying Li (cherry picked from commit 7fc0c4f3a35bd7d9e8c26f6d86acf7e2718eb778) Signed-off-by: Sebastiaan van Stijn --- .circleci/config.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eb83412c5b..8c651e05fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,12 @@ version: 2 jobs: build: - working_directory: ~/go/src/github.com/docker/swarmkit + # CircleCI by default sets the gopath to be ~/.go_workspace and /usr/local/go_workspace + # apparently. We cannot set the working directory or environment variables by using + # other environment variables (although the working directory can use the `~` character, + # but environment variables cannot), so to avoid having to override the GOPATH for every + # run command, just hard code in the directory to be CircleCI expects it to be. + working_directory: /home/circleci/.go_workspace/src/github.com/docker/swarmkit environment: # Needed to install go OS: linux @@ -44,7 +49,6 @@ jobs: name: Install go command: | sudo rm -rf /usr/local/go - rm -rf "$GOPATH" curl -fsSL -o "$HOME/go.tar.gz" "https://storage.googleapis.com/golang/go$GOVERSION.$OS-$ARCH.tar.gz" sudo tar -C /usr/local -xzf "$HOME/go.tar.gz" @@ -79,7 +83,7 @@ jobs: # The GOPATH setting would not be needed if we used the golang docker image - run: name: Compile/lint/vet/protobuf validation - command: GOPATH="$HOME/go" make check binaries checkprotos + command: make check binaries checkprotos - run: name: Run unit tests @@ -87,13 +91,13 @@ jobs: sudo mkdir /tmpfs sudo mount -t tmpfs tmpfs /tmpfs sudo chown 1000:1000 /tmpfs - GOPATH="$HOME/go" TMPDIR=/tmpfs make coverage + TMPDIR=/tmpfs make coverage - run: name: Run integration tests command: | # TMPFS has already been set up previously in the unit test step - GOPATH="$HOME/go" TMPDIR=/tmpfs make coverage-integration + TMPDIR=/tmpfs make coverage-integration - run: name: Push coverage info to codecov.io From 8e68ca8a60320abb0f20d537c5fab6797dc90490 Mon Sep 17 00:00:00 2001 From: Vincent Robbemond Date: Mon, 5 Mar 2018 03:15:11 +0100 Subject: [PATCH 9/9] Fix possible data race in manager/state/store/memory_test.go Signed-off-by: Vincent Robbemond (cherry picked from commit 06add13d57411618c18d4b9d181013e932462834) Signed-off-by: Sebastiaan van Stijn --- manager/state/store/memory_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manager/state/store/memory_test.go b/manager/state/store/memory_test.go index b4df214b0f..cd7740e734 100644 --- a/manager/state/store/memory_test.go +++ b/manager/state/store/memory_test.go @@ -2035,7 +2035,7 @@ func BenchmarkNodeConcurrency(b *testing.B) { var wg sync.WaitGroup for c := 0; c != 5; c++ { wg.Add(1) - go func() { + go func(c int) { defer wg.Done() for i := 0; i < b.N; i++ { _ = s.Update(func(tx1 Tx) error { @@ -2050,7 +2050,7 @@ func BenchmarkNodeConcurrency(b *testing.B) { return nil }) } - }() + }(c) } for c := 0; c != 5; c++ {