From dd2a01e730fd7e40997544039cbecbfb72130bd4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 29 Jun 2022 13:18:20 -0600 Subject: [PATCH 1/9] .gitignore: Begone with pointless **/ Entries in .gitignore aren't rooted unless they start with "/", so a leading "**/" is pointless. Signed-off-by: Luke Shumaker --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5ed1a0c03d..b5731b364f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -**/.DS_Store +.DS_Store bin/ From 2e59fcb9919b52835dcfe7cc84657353177d85a4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 5 Jul 2022 20:23:47 -0600 Subject: [PATCH 2/9] build-sys: Turn off intermediate file removal Signed-off-by: Luke Shumaker --- tools/make/common.mk | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/make/common.mk b/tools/make/common.mk index 0abd3a11ad..053fce9ca0 100644 --- a/tools/make/common.mk +++ b/tools/make/common.mk @@ -2,6 +2,18 @@ # # All make targets related to common variables are defined in this file. +# ==================================================================================================== +# Configure Make itself: +# ==================================================================================================== + +# Turn off .INTERMEDIATE file removal by marking all files as +# .SECONDARY. .INTERMEDIATE file removal is a space-saving hack from +# a time when drives were small; on modern computers with plenty of +# storage, it causes nothing but headaches. +# +# https://news.ycombinator.com/item?id=16486331 +.SECONDARY: + # ==================================================================================================== # ROOT Options: # ==================================================================================================== From c68457cd295a8897758d968ec87eb08a8b320a59 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 29 Jun 2022 12:00:44 -0600 Subject: [PATCH 3/9] build-sys: Don't use recursive Make Arko had trouble with this, because variables were being used before they were defined. In order to make those errors easy to debug, I tacked on --warn-undefined-variables so that these things get called out. Signed-off-by: Luke Shumaker --- Makefile | 2 +- tools/docker/builder/Dockerfile | 2 +- tools/make/common.mk | 16 ++++++++-------- tools/make/golang.mk | 16 +++++++--------- tools/make/image.mk | 17 ++++++++--------- tools/make/lint.mk | 2 +- 6 files changed, 26 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index e2b353051d..0c47a6f41a 100644 --- a/Makefile +++ b/Makefile @@ -27,5 +27,5 @@ ifeq ($(MAKE_IN_DOCKER), 1) @$(DOCKER_RUN_CMD) $(DOCKER_BUILDER_IMAGE):$(DOCKER_BUILDER_TAG) MAKE_IN_DOCKER=0 $@ else # Run make locally - @$(MAKE) -f tools/make/common.mk $@ + @$(MAKE) --warn-undefined-variables -f tools/make/common.mk $@ endif diff --git a/tools/docker/builder/Dockerfile b/tools/docker/builder/Dockerfile index 4de6a80752..522937f84e 100644 --- a/tools/docker/builder/Dockerfile +++ b/tools/docker/builder/Dockerfile @@ -23,4 +23,4 @@ RUN python3 -m pip install --no-cache-dir yamllint==${YAMLLINT_VERSION} RUN python3 -m pip install --no-cache-dir codespell==${CODESPELL_VERSION} WORKDIR /workspace -ENTRYPOINT ["make"] \ No newline at end of file +ENTRYPOINT ["make", "--warn-undefined-variables"] diff --git a/tools/make/common.mk b/tools/make/common.mk index 053fce9ca0..919b799a3a 100644 --- a/tools/make/common.mk +++ b/tools/make/common.mk @@ -20,14 +20,6 @@ ROOT_PACKAGE=github.com/envoyproxy/gateway -# ==================================================================================================== -# Includes: -# ==================================================================================================== -include tools/make/golang.mk -include tools/make/image.mk -include tools/make/lint.mk -include tools/make/kube.mk - # Set Root Directory Path ifeq ($(origin ROOT_DIR),undefined) ROOT_DIR := $(abspath $(shell pwd -P)) @@ -78,6 +70,14 @@ ifeq (${BINS},) $(error Could not determine BINS, set ROOT_DIR or run in source dir) endif +# ==================================================================================================== +# Includes: +# ==================================================================================================== +include tools/make/golang.mk +include tools/make/image.mk +include tools/make/lint.mk +include tools/make/kube.mk + # Log the running target LOG_TARGET = echo "===========> Running $@..." # Log debugging info diff --git a/tools/make/golang.mk b/tools/make/golang.mk index 81ef821daa..2496790097 100644 --- a/tools/make/golang.mk +++ b/tools/make/golang.mk @@ -24,14 +24,12 @@ go.build.%: # Build the envoy-gateway binaries in the hosted platforms. .PHONY: go.build -go.build: - @$(MAKE) $(addprefix go.build., $(addprefix $(PLATFORM)., $(BINS))) +go.build: $(addprefix go.build., $(addprefix $(PLATFORM)., $(BINS))) # Build the envoy-gateway binaries in multi platforms # It will build the linux/amd64, linux/arm64, darwin/amd64, darwin/arm64 binaries out. .PHONY: go.build.multiarch -go.build.multiarch: - @$(MAKE) $(foreach p,$(PLATFORMS),$(addprefix go.build., $(addprefix $(p)., $(BINS)))) +go.build.multiarch: $(foreach p,$(PLATFORMS),$(addprefix go.build., $(addprefix $(p)., $(BINS)))) .PHONY: go.test.unit go.test.unit: ## Run go unit tests @@ -58,20 +56,20 @@ go.tidy: .PHONY: build build: ## Build envoy-gateway for host platform. See Option PLATFORM and BINS. - @$(MAKE) go.build +build: go.build .PHONY: build-multiarch build-multiarch: ## Build envoy-gateway for multiple platforms. See Option PLATFORMS and IMAGES. - @$(MAKE) go.build.multiarch +build-multiarch: go.build.multiarch .PHONY: test test: ## Run all Go test of code sources. - @$(MAKE) go.test.unit +test: go.test.unit .PHONY: format format: ## Update dependences with mod tidy. - @$(MAKE) go.tidy +format: go.tidy .PHONY: clean clean: ## Remove all files that are created during builds. - @$(MAKE) go.clean +clean: go.clean diff --git a/tools/make/image.mk b/tools/make/image.mk index ee99172f6e..311c56e08c 100644 --- a/tools/make/image.mk +++ b/tools/make/image.mk @@ -40,11 +40,10 @@ image.verify: fi .PHONY: image.build -image.build: image.verify - @$(MAKE) $(addprefix image.build., $(addprefix $(IMAGE_PLAT)., $(IMAGES))) +image.build: $(addprefix image.build.$(IMAGE_PLAT)., $(IMAGES)) .PHONY: image.build.% -image.build.%: go.build.% +image.build.%: go.build.% image.verify $(eval IMAGES := $(COMMAND)) $(eval IMAGE_PLAT := $(subst _,/,$(PLATFORM))) @$(call log, "Building image $(IMAGES) in tag $(TAG) for $(IMAGE_PLAT)" @@ -53,10 +52,10 @@ image.build.%: go.build.% $(DOCKER) build --platform $(IMAGE_PLAT) $(BUILD_SUFFIX) .PHONY: image.push -image.push: image.verify $(addprefix image.push., $(addprefix $(IMAGE_PLAT)., $(IMAGES))) +image.push: $(addprefix image.push.$(IMAGE_PLAT)., $(IMAGES)) .PHONY: image.push.% -image.push.%: +image.push.%: image.build.% $(eval COMMAND := $(word 2,$(subst ., ,$*))) $(eval IMAGES := $(COMMAND)) $(eval PLATFORM := $(word 1,$(subst ., ,$*))) @@ -96,16 +95,16 @@ image.push.multiarch: image.multiarch.setup go.build.multiarch .PHONY: image image: ## Build docker images for host platform. See Option PLATFORM and BINS. - @$(MAKE) image.build +image: image.build .PHONY: image-multiarch image-multiarch: ## Build docker images for multiple platforms. See Option PLATFORMS and IMAGES. - @$(MAKE) image.build.multiarch +image-multiarch: image.build.multiarch .PHONY: push push: ## Push docker images to registry. - @$(MAKE) image.push +push: image.push .PHONY: push-multiarch push-multiarch: ## Push docker images for multiple platforms to registry. - @$(MAKE) image.push.multiarch +push-multiarch: image.push.multiarch diff --git a/tools/make/lint.mk b/tools/make/lint.mk index 0812ff4503..11d3e9cc36 100644 --- a/tools/make/lint.mk +++ b/tools/make/lint.mk @@ -6,7 +6,7 @@ .PHONY: lint lint: ## Run all linter of code sources, including golint, yamllint, whitenoise lint and codespell. - @$(MAKE) lint.golint lint.yamllint lint.codespell lint.whitenoise +lint: lint.golint lint.yamllint lint.codespell lint.whitenoise .PHONY: lint.golint lint.golint: From 6ad77609797ddd54d49cae9152a743254c0162ba Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 29 Jun 2022 12:03:56 -0600 Subject: [PATCH 4/9] build-sys: Don't hide what commands are running Signed-off-by: Luke Shumaker --- tools/make/golang.mk | 10 +++++----- tools/make/image.mk | 10 +++++----- tools/make/lint.mk | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/make/golang.mk b/tools/make/golang.mk index 2496790097..32f83d14a5 100644 --- a/tools/make/golang.mk +++ b/tools/make/golang.mk @@ -20,7 +20,7 @@ go.build.%: $(eval OS := $(word 1,$(subst _, ,$(PLATFORM)))) $(eval ARCH := $(word 2,$(subst _, ,$(PLATFORM)))) @$(call log, "Building binary $(COMMAND) with commit $(REV) in version $(VERSION) for $(OS) $(ARCH)") - @CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o $(OUTPUT_DIR)/$(OS)/$(ARCH)/$(COMMAND) $(ROOT_PACKAGE)/cmd/$(COMMAND) + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o $(OUTPUT_DIR)/$(OS)/$(ARCH)/$(COMMAND) $(ROOT_PACKAGE)/cmd/$(COMMAND) # Build the envoy-gateway binaries in the hosted platforms. .PHONY: go.build @@ -33,21 +33,21 @@ go.build.multiarch: $(foreach p,$(PLATFORMS),$(addprefix go.build., $(addprefix .PHONY: go.test.unit go.test.unit: ## Run go unit tests - @go test ./... + go test ./... .PHONY: go.test.coverage go.test.coverage: ## Run go unit tests in GitHub Actions - @go test ./... -race -coverprofile=coverage.xml -covermode=atomic + go test ./... -race -coverprofile=coverage.xml -covermode=atomic .PHONY: go.clean go.clean: ## Clean the building output files @$(call log, "Cleaning all build output") - @rm -rf $(OUTPUT_DIR) + rm -rf $(OUTPUT_DIR) .PHONY: go.tidy go.tidy: @$(LOG_TARGET) - @go mod tidy -compat=$(GO_VERSION) + go mod tidy -compat=$(GO_VERSION) ## ensure all changes have been committed git diff --exit-code go.mod git diff --exit-code go.sum diff --git a/tools/make/image.mk b/tools/make/image.mk index 311c56e08c..d94c1fa991 100644 --- a/tools/make/image.mk +++ b/tools/make/image.mk @@ -68,20 +68,20 @@ image.push.%: image.build.% .PHONY: image.multiarch.verify image.multiarch.verify: $(eval pass := $(shell )) - @docker buildx --help | grep -qw "docker buildx" || { \ + docker buildx --help | grep -qw "docker buildx" || { \ echo "Cannot find `docker buildx`, please install first"; \ exit 1; } .PHONY: image.multiarch.emulate $(EMULATE_TARGETS) image.multiarch.emulate: $(EMULATE_TARGETS) $(EMULATE_TARGETS): image.multiarch.emulate.%: - @docker run --rm --privileged tonistiigi/binfmt --install linux/$* # Install QEMU emulator, the same emulator as the host will report an error but can safe ignore + docker run --rm --privileged tonistiigi/binfmt --install linux/$* # Install QEMU emulator, the same emulator as the host will report an error but can safe ignore .PHONY: image.multiarch.setup image.multiarch.setup: image.verify image.multiarch.verify image.multiarch.emulate - @docker run --rm --privileged tonistiigi/binfmt --install all # Install QEMU emulators - @docker buildx rm $(BUILDX_CONTEXT) || : - @docker buildx create --use --name $(BUILDX_CONTEXT) --platform "${BUILDX_PLATFORMS}" + docker run --rm --privileged tonistiigi/binfmt --install all # Install QEMU emulators + docker buildx rm $(BUILDX_CONTEXT) || : + docker buildx create --use --name $(BUILDX_CONTEXT) --platform "${BUILDX_PLATFORMS}" .PHONY: image.build.multiarch image.build.multiarch: image.multiarch.setup go.build.multiarch diff --git a/tools/make/lint.mk b/tools/make/lint.mk index 11d3e9cc36..d5805ad909 100644 --- a/tools/make/lint.mk +++ b/tools/make/lint.mk @@ -11,18 +11,18 @@ lint: lint.golint lint.yamllint lint.codespell lint.whitenoise .PHONY: lint.golint lint.golint: @echo Running Go linter ... - @golangci-lint run --build-tags=e2e --config=tools/linter/golangci-lint/.golangci.yml + golangci-lint run --build-tags=e2e --config=tools/linter/golangci-lint/.golangci.yml .PHONY: lint.yamllint lint.yamllint: @echo Running YAML linter ... - @yamllint --config-file=tools/linter/yamllint/.yamllint changelogs/ + yamllint --config-file=tools/linter/yamllint/.yamllint changelogs/ .PHONY: lint.codespell lint.codespell: CODESPELL_SKIP := $(shell cat tools/linter/codespell/.codespell.skip | tr \\n ',') lint.codespell: @echo Running Codespell linter ... - @codespell --skip $(CODESPELL_SKIP) --ignore-words tools/linter/codespell/.codespell.ignorewords --check-filenames --check-hidden -q2 + codespell --skip $(CODESPELL_SKIP) --ignore-words tools/linter/codespell/.codespell.ignorewords --check-filenames --check-hidden -q2 .PHONY: lint.whitenoise lint.whitenoise: From 374d7760a699033ff19a9a813df4ad565e74f871 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 5 Jul 2022 16:05:06 -0600 Subject: [PATCH 5/9] .github: Refactor a bit, splitting things up Signed-off-by: Luke Shumaker --- .github/workflows/build_and_test.yaml | 54 ++++++++++----------- tools/github-actions/setup-deps/action.yaml | 12 +++++ 2 files changed, 37 insertions(+), 29 deletions(-) create mode 100644 tools/github-actions/setup-deps/action.yaml diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 2ce4fb17d9..145350cfc5 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -12,16 +12,14 @@ on: - "release-v*" paths-ignore: - "**/*.png" -env: - GO_VERSION: 1.18.2 - ENVOY_GATEWAY_DEV_IMAGE: envoyproxy/gateway-dev - ENVOY_GATEWAY_DEV_TAG: latest jobs: - build-and-test: + lint: runs-on: ubuntu-latest steps: - - name: Check out code - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + - uses: ./tools/github-actions/setup-deps + + # lint - name: Run whitenoise lint run: | MAKE_IN_DOCKER=1 make lint.whitenoise @@ -31,17 +29,21 @@ jobs: - name: Run codespell run: | MAKE_IN_DOCKER=1 make lint.codespell - - name: Run golangci-lint + - name: Run golangci-lint run: | MAKE_IN_DOCKER=1 make lint.golint - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: ${{ env.GO_VERSION }} - - name: Build Binaries - run: make build-multiarch - - name: Test and report coverage - run: make go.test.coverage + + build-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./tools/github-actions/setup-deps + + # build + - run: make build-multiarch + + # test + - run: make go.test.coverage - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 with: @@ -49,31 +51,25 @@ jobs: files: ./coverage.xml name: codecov-envoy-gateway verbose: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build Docker Image - uses: docker/build-push-action@v3 - with: - file: ./tools/docker/envoy-gateway/Dockerfile - context: bin - platforms: linux/amd64 - tags: ${{ env.ENVOY_GATEWAY_DEV_IMAGE }}:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + + # push - name: Login to DockerHub if: github.event_name == 'push' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Push to envoyproxy/gateway-dev + - run: make push-multiarch + if: github.event_name == 'push' + - name: Push to :latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' + # TODO(lukeshu): Switch this to be a Make target uses: docker/build-push-action@v3 with: file: ./tools/docker/envoy-gateway/Dockerfile context: bin platforms: linux/amd64,linux/arm64 push: true - tags: ${{ env.ENVOY_GATEWAY_DEV_IMAGE }}:${{ github.sha }}, ${{ env.ENVOY_GATEWAY_DEV_IMAGE }}:${{ env.ENVOY_GATEWAY_DEV_TAG }} + tags: docker.io/envoyproxy/gateway-dev:latest cache-from: type=gha cache-to: type=gha,mode=max diff --git a/tools/github-actions/setup-deps/action.yaml b/tools/github-actions/setup-deps/action.yaml new file mode 100644 index 0000000000..cb389e0dbf --- /dev/null +++ b/tools/github-actions/setup-deps/action.yaml @@ -0,0 +1,12 @@ +name: setup-deps +description: Install host system dependencies + +runs: + using: composite + steps: + - uses: actions/setup-go@v3 + with: + go-version: 1.18.2 + cache: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 From cbb804d35903f3283ac463326d82483ddc084d6f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 5 Jul 2022 16:14:54 -0600 Subject: [PATCH 6/9] build-sys, .github: Have CI use the combined `make lint` Rather tha calling each individually. Signed-off-by: Luke Shumaker --- .github/workflows/build_and_test.yaml | 15 +++------------ Makefile | 1 + 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 145350cfc5..222efe4259 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -20,18 +20,9 @@ jobs: - uses: ./tools/github-actions/setup-deps # lint - - name: Run whitenoise lint - run: | - MAKE_IN_DOCKER=1 make lint.whitenoise - - name: Run yamllint - run: | - MAKE_IN_DOCKER=1 make lint.yamllint - - name: Run codespell - run: | - MAKE_IN_DOCKER=1 make lint.codespell - - name: Run golangci-lint - run: | - MAKE_IN_DOCKER=1 make lint.golint + - run: make -k lint + env: + MAKE_IN_DOCKER: "1 " build-and-test: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 0c47a6f41a..c072f39a3a 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ DOCKER_BUILD_CMD ?= DOCKER_BUILDKIT=1 docker build DOCKER_RUN_CMD ?= docker run \ --rm \ -t \ + --env=MAKEFLAGS \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ${PWD}:/workspace From 77471cc25022b87b504395f69d6c920895eb8944 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 29 Jun 2022 15:23:43 -0600 Subject: [PATCH 7/9] build-sys: make lint: Restore integration with GitHub Actions annotations This was dropped in 80b78e5521a0ce32274c36a845da3955df4aec09 AKA https://github.com/envoyproxy/gateway/pull/113 Signed-off-by: Luke Shumaker --- Makefile | 4 +++- tools/linter/codespell/matcher.json | 26 ++++++++++++++++++++++++++ tools/make/lint.mk | 23 +++++++++++++++++++++-- 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 tools/linter/codespell/matcher.json diff --git a/Makefile b/Makefile index c072f39a3a..b091789f10 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,10 @@ DOCKER_RUN_CMD ?= docker run \ --rm \ -t \ --env=MAKEFLAGS \ + --env=GITHUB_ACTION \ -v /var/run/docker.sock:/var/run/docker.sock \ - -v ${PWD}:/workspace + --volume=$(CURDIR):$(CURDIR) \ + --workdir=$(CURDIR) %: ifeq ($(MAKE_IN_DOCKER), 1) diff --git a/tools/linter/codespell/matcher.json b/tools/linter/codespell/matcher.json new file mode 100644 index 0000000000..cded56c503 --- /dev/null +++ b/tools/linter/codespell/matcher.json @@ -0,0 +1,26 @@ +{ + "problemMatcher": [ + { + "owner": "codespell-matcher-default", + "pattern": [ + { + "regexp": "^(.+):(\\d+):\\s+(.+)$", + "file": 1, + "line": 2, + "message": 3 + } + ] + }, + { + "owner": "codespell-matcher-specified", + "pattern": [ + { + "regexp": "^(ERROR|WARNING):\\s+(.+):\\s+(.+?)\\s*$", + "file": 3, + "severity": 1, + "message": 2 + } + ] + } + ] +} diff --git a/tools/make/lint.mk b/tools/make/lint.mk index d5805ad909..bfb6a4ff65 100644 --- a/tools/make/lint.mk +++ b/tools/make/lint.mk @@ -4,25 +4,44 @@ ##@ Lint +GITHUB_ACTION ?= + .PHONY: lint lint: ## Run all linter of code sources, including golint, yamllint, whitenoise lint and codespell. lint: lint.golint lint.yamllint lint.codespell lint.whitenoise +GOLANGCI_LINT_FLAGS ?= $(if $(GITHUB_ACTION),--out-format=github-actions) .PHONY: lint.golint lint.golint: @echo Running Go linter ... - golangci-lint run --build-tags=e2e --config=tools/linter/golangci-lint/.golangci.yml + golangci-lint run $(GOLANGCI_LINT_FLAGS) --build-tags=e2e --config=tools/linter/golangci-lint/.golangci.yml .PHONY: lint.yamllint lint.yamllint: @echo Running YAML linter ... yamllint --config-file=tools/linter/yamllint/.yamllint changelogs/ +CODESPELL_FLAGS ?= $(if $(GITHUB_ACTION),--disable-colors) .PHONY: lint.codespell lint.codespell: CODESPELL_SKIP := $(shell cat tools/linter/codespell/.codespell.skip | tr \\n ',') lint.codespell: @echo Running Codespell linter ... - codespell --skip $(CODESPELL_SKIP) --ignore-words tools/linter/codespell/.codespell.ignorewords --check-filenames --check-hidden -q2 +# This ::add-matcher/::remove-matcher business is based on +# https://github.com/codespell-project/actions-codespell/blob/2292753ad350451611cafcbabc3abe387491339a/entrypoint.sh +# We do this here instead of just using +# codespell-project/codespell-problem-matcher@v1 so that the matcher +# doesn't apply to the other linters that `make lint` also runs. +# +# This recipe is written a little awkwardly with everything running in +# one shell, this is because we want the ::remove-matcher lines to get +# printed whether or not it finds complaints. + @PS4=; set -e; { \ + if test -n "$$GITHUB_ACTION"; then \ + printf '::add-matcher::$(CURDIR)/tools/linter/codespell/matcher.json\n'; \ + trap "printf '::remove-matcher owner=codespell-matcher-default::\n::remove-matcher owner=codespell-matcher-specified::\n'" EXIT; \ + fi; \ + (set -x; codespell $(CODESPELL_FLAGS) --skip $(CODESPELL_SKIP) --ignore-words tools/linter/codespell/.codespell.ignorewords --check-filenames --check-hidden -q2); \ + } .PHONY: lint.whitenoise lint.whitenoise: From a0cb1bde345620d4b864ea900a32f01d876f92de Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 29 Jun 2022 15:23:43 -0600 Subject: [PATCH 8/9] build-sys: Introduce tools.mk to manage required tools Signed-off-by: Luke Shumaker --- .github/workflows/build_and_test.yaml | 1 + DEVELOPER.md | 21 +- tools/linter/codespell/.codespell.skip | 3 +- tools/make/common.mk | 1 + tools/make/kube.mk | 52 +- tools/make/lint.mk | 28 +- tools/make/tools.mk | 32 + tools/src/codespell/requirements.txt | 1 + tools/src/controller-gen/go.mod | 39 + tools/src/controller-gen/go.sum | 915 +++++++++++ tools/src/controller-gen/pin.go | 6 + tools/src/golangci-lint/go.mod | 165 ++ tools/src/golangci-lint/go.sum | 1414 +++++++++++++++++ tools/src/golangci-lint/pin.go | 6 + tools/src/kustomize/go.mod | 94 ++ tools/src/kustomize/go.sum | 599 +++++++ tools/src/kustomize/pin.go | 6 + tools/src/setup-envtest/go.mod | 22 + tools/src/setup-envtest/go.sum | 161 ++ tools/src/setup-envtest/pin.go | 6 + .../lint-whitenoise => src/whitenoise.sh} | 0 tools/src/yamllint/requirements.txt | 1 + 22 files changed, 3508 insertions(+), 65 deletions(-) create mode 100644 tools/make/tools.mk create mode 100644 tools/src/codespell/requirements.txt create mode 100644 tools/src/controller-gen/go.mod create mode 100644 tools/src/controller-gen/go.sum create mode 100644 tools/src/controller-gen/pin.go create mode 100644 tools/src/golangci-lint/go.mod create mode 100644 tools/src/golangci-lint/go.sum create mode 100644 tools/src/golangci-lint/pin.go create mode 100644 tools/src/kustomize/go.mod create mode 100644 tools/src/kustomize/go.sum create mode 100644 tools/src/kustomize/pin.go create mode 100644 tools/src/setup-envtest/go.mod create mode 100644 tools/src/setup-envtest/go.sum create mode 100644 tools/src/setup-envtest/pin.go rename tools/{linter/lint-whitenoise => src/whitenoise.sh} (100%) mode change 100755 => 100644 create mode 100644 tools/src/yamllint/requirements.txt diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 222efe4259..9cf9a43f9f 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -20,6 +20,7 @@ jobs: - uses: ./tools/github-actions/setup-deps # lint + - run: make lint-deps - run: make -k lint env: MAKE_IN_DOCKER: "1 " diff --git a/DEVELOPER.md b/DEVELOPER.md index 0a7ff3ac5c..43a4680e4f 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -10,25 +10,20 @@ Envoy Gateway is built using a [make][make]-based build system. Our CI is based * Installation Guide: https://go.dev/doc/install ### make -* Recommended Version: 4.3 -* Installation Guide: https://www.gnu.org/software/make. +* Recommended Version: 4.0 or later +* Installation Guide: https://www.gnu.org/software/make ### docker * Optional when you want to build a Docker image or run `make` inside Docker. * Recommended Version: 20.10.16 * Installation Guide: https://docs.docker.com/engine/install -### golangci-lint -* Recommended Version: 1.46.2 -* Installation Guide: https://github.com/golangci/golangci-lint#install - -### yamllint -* Recommended Version: 1.24.2 -* Installation Guide: https://yamllint.readthedocs.io/en/stable/quickstart.html#installing-yamllint - -### codespell -* Recommended Version: 2.1.0 -* Installation Guide: https://github.com/codespell-project/codespell#installation +### python3 +* Need a `python3` program +* Must have a functioning `venv` module; this is part of the standard + library, but some distributions (such as Debian and Ubuntu) replace + it with a stub and require you to install a `python3-venv` package + separately. __Note:__ If you do not have these tools installed on your machine, you can alternatively run `MAKE_IN_DOCKER=1 make ` to run `make` inside a Docker container which has all the preinstalled tools needed to diff --git a/tools/linter/codespell/.codespell.skip b/tools/linter/codespell/.codespell.skip index 12f7525da8..0f7ff043e1 100644 --- a/tools/linter/codespell/.codespell.skip +++ b/tools/linter/codespell/.codespell.skip @@ -9,4 +9,5 @@ *.ico *.svg go.mod -go.sum \ No newline at end of file +go.sum +bin diff --git a/tools/make/common.mk b/tools/make/common.mk index 919b799a3a..ae20221f28 100644 --- a/tools/make/common.mk +++ b/tools/make/common.mk @@ -73,6 +73,7 @@ endif # ==================================================================================================== # Includes: # ==================================================================================================== +include tools/make/tools.mk include tools/make/golang.mk include tools/make/image.mk include tools/make/lint.mk diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 66750f308e..84c08e9eaa 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -1,16 +1,16 @@ ##@ Kubernetes Development .PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=envoy-gateway-role crd webhook paths="./..." output:crd:artifacts:config=pkg/provider/kubernetes/config/crd/bases +manifests: $(tools/controller-gen) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(tools/controller-gen) rbac:roleName=envoy-gateway-role crd webhook paths="./..." output:crd:artifacts:config=pkg/provider/kubernetes/config/crd/bases .PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object paths="./..." +generate: $(tools/controller-gen) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(tools/controller-gen) object paths="./..." .PHONY: kube-test -kube-test: manifests generate lint envtest ## Run Kubernetes provider tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out +kube-test: manifests generate lint $(tools/setup-envtest) ## Run Kubernetes provider tests. + KUBEBUILDER_ASSETS="$(shell $(tools/setup-envtest) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out ##@ Kubernetes Deployment @@ -20,41 +20,9 @@ ifndef ignore-not-found endif .PHONY: kube-install -kube-install: manifests kustomize ## Install CRDs into the Kubernetes cluster specified in ~/.kube/config. - $(KUSTOMIZE) build pkg/provider/kubernetes/config/crd | kubectl apply -f - +kube-install: manifests $(tools/kustomize) ## Install CRDs into the Kubernetes cluster specified in ~/.kube/config. + $(tools/kustomize) build pkg/provider/kubernetes/config/crd | kubectl apply -f - .PHONY: kube-uninstall -kube-uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build pkg/provider/kubernetes/config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - -##@ Kubernetes Build Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -## Tool Binaries -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest - -## Tool Versions -KUSTOMIZE_VERSION ?= v3.8.7 -CONTROLLER_TOOLS_VERSION ?= v0.8.0 - -KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN) - -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. -$(CONTROLLER_GEN): $(LOCALBIN) - GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) - -.PHONY: envtest -envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. -$(ENVTEST): $(LOCALBIN) - GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest +kube-uninstall: manifests $(tools/kustomize) ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(tools/kustomize) build pkg/provider/kubernetes/config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - diff --git a/tools/make/lint.mk b/tools/make/lint.mk index bfb6a4ff65..dddd4cbbf6 100644 --- a/tools/make/lint.mk +++ b/tools/make/lint.mk @@ -8,23 +8,31 @@ GITHUB_ACTION ?= .PHONY: lint lint: ## Run all linter of code sources, including golint, yamllint, whitenoise lint and codespell. -lint: lint.golint lint.yamllint lint.codespell lint.whitenoise + +PHONY: lint-deps +lint-deps: ## Everything necessary to lint (useful to separate out in the logs) GOLANGCI_LINT_FLAGS ?= $(if $(GITHUB_ACTION),--out-format=github-actions) .PHONY: lint.golint -lint.golint: +lint: lint.golint +lint-deps: $(tools/golangci-lint) +lint.golint: $(tools/golangci-lint) @echo Running Go linter ... - golangci-lint run $(GOLANGCI_LINT_FLAGS) --build-tags=e2e --config=tools/linter/golangci-lint/.golangci.yml + $(tools/golangci-lint) run $(GOLANGCI_LINT_FLAGS) --build-tags=e2e --config=tools/linter/golangci-lint/.golangci.yml .PHONY: lint.yamllint -lint.yamllint: +lint: lint.yamllint +lint-deps: $(tools/yamllint) +lint.yamllint: $(tools/yamllint) @echo Running YAML linter ... - yamllint --config-file=tools/linter/yamllint/.yamllint changelogs/ + $(tools/yamllint) --config-file=tools/linter/yamllint/.yamllint changelogs/ CODESPELL_FLAGS ?= $(if $(GITHUB_ACTION),--disable-colors) .PHONY: lint.codespell +lint: lint.codespell +lint-deps: $(tools/codespell) lint.codespell: CODESPELL_SKIP := $(shell cat tools/linter/codespell/.codespell.skip | tr \\n ',') -lint.codespell: +lint.codespell: $(tools/codespell) @echo Running Codespell linter ... # This ::add-matcher/::remove-matcher business is based on # https://github.com/codespell-project/actions-codespell/blob/2292753ad350451611cafcbabc3abe387491339a/entrypoint.sh @@ -40,10 +48,12 @@ lint.codespell: printf '::add-matcher::$(CURDIR)/tools/linter/codespell/matcher.json\n'; \ trap "printf '::remove-matcher owner=codespell-matcher-default::\n::remove-matcher owner=codespell-matcher-specified::\n'" EXIT; \ fi; \ - (set -x; codespell $(CODESPELL_FLAGS) --skip $(CODESPELL_SKIP) --ignore-words tools/linter/codespell/.codespell.ignorewords --check-filenames --check-hidden -q2); \ + (set -x; $(tools/codespell) $(CODESPELL_FLAGS) --skip $(CODESPELL_SKIP) --ignore-words tools/linter/codespell/.codespell.ignorewords --check-filenames --check-hidden -q2); \ } .PHONY: lint.whitenoise -lint.whitenoise: +lint: lint.whitenoise +lint-deps: $(tools/whitenoise) +lint.whitenoise: $(tools/whitenoise) @echo Running WhiteNoise linter ... - tools/linter/lint-whitenoise + $(tools/whitenoise) diff --git a/tools/make/tools.mk b/tools/make/tools.mk new file mode 100644 index 0000000000..36b7b15343 --- /dev/null +++ b/tools/make/tools.mk @@ -0,0 +1,32 @@ +tools.bindir = tools/bin +tools.srcdir = tools/src + +# Shell scripts +# ============= +# +tools/whitenoise = $(tools.bindir)/whitenoise +$(tools.bindir)/%: $(tools.srcdir)/%.sh + mkdir -p $(@D) + install $< $@ + +# `go get`-able things +# ==================== +# +tools/controller-gen = $(tools.bindir)/controller-gen +tools/golangci-lint = $(tools.bindir)/golangci-lint +tools/kustomize = $(tools.bindir)/kustomize +tools/setup-envtest = $(tools.bindir)/setup-envtest +$(tools.bindir)/%: $(tools.srcdir)/%/pin.go $(tools.srcdir)/%/go.mod + cd $( Date: Wed, 29 Jun 2022 11:51:28 -0600 Subject: [PATCH 9/9] build-sys: Begone with MAKE_IN_DOCKER Signed-off-by: Luke Shumaker --- .github/workflows/build_and_test.yaml | 2 -- DEVELOPER.md | 4 --- Makefile | 42 +++++++++------------------ tools/docker/builder/Dockerfile | 26 ----------------- 4 files changed, 14 insertions(+), 60 deletions(-) delete mode 100644 tools/docker/builder/Dockerfile diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 9cf9a43f9f..e70fe001cc 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -22,8 +22,6 @@ jobs: # lint - run: make lint-deps - run: make -k lint - env: - MAKE_IN_DOCKER: "1 " build-and-test: runs-on: ubuntu-latest diff --git a/DEVELOPER.md b/DEVELOPER.md index 43a4680e4f..19ef87234f 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -25,10 +25,6 @@ Envoy Gateway is built using a [make][make]-based build system. Our CI is based it with a stub and require you to install a `python3-venv` package separately. -__Note:__ If you do not have these tools installed on your machine, you can alternatively run -`MAKE_IN_DOCKER=1 make ` to run `make` inside a Docker container which has all the preinstalled tools needed to -support all the `make` targets. - ## Quick start Run `make help` to see all the available targets to build, test and run `envoy-gateway`. diff --git a/Makefile b/Makefile index b091789f10..7a47bc2b61 100644 --- a/Makefile +++ b/Makefile @@ -3,32 +3,18 @@ # Supported Targets: (Run `make help` to see more information) # ==================================================================================================== -# This is a wrapper around `make` so it can run -# directly on the host or inside a container -# -# Set MAKE_IN_DOCKER=1 as an environment variable to run `make` inside -# a Docker container with preinstalled tools. +# This file is a wrapper around `make` so that we can force on the +# --warn-undefined-variables flag. Sure, you can set +# `MAKEFLAGS += --warn-undefined-variables` from inside of a Makefile, +# but then it won't turn on until the second phase (recipe execution), +# and won't actually be on during the initial phase (parsing). +# See: https://www.gnu.org/software/make/manual/make.html#Reading-Makefiles -DOCKER_BUILDER_IMAGE ?= envoyproxy/gateway-dev-builder -DOCKER_BUILDER_TAG ?= latest -DOCKER_BUILD_CMD ?= DOCKER_BUILDKIT=1 docker build -DOCKER_RUN_CMD ?= docker run \ - --rm \ - -t \ - --env=MAKEFLAGS \ - --env=GITHUB_ACTION \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --volume=$(CURDIR):$(CURDIR) \ - --workdir=$(CURDIR) - -%: -ifeq ($(MAKE_IN_DOCKER), 1) -# Build builder image - @$(DOCKER_BUILD_CMD) -t $(DOCKER_BUILDER_IMAGE):$(DOCKER_BUILDER_TAG) - < tools/docker/builder/Dockerfile -# Run make inside the builder image -# Run with MAKE_IN_DOCKER=0 to eliminate an infinite loop - @$(DOCKER_RUN_CMD) $(DOCKER_BUILDER_IMAGE):$(DOCKER_BUILDER_TAG) MAKE_IN_DOCKER=0 $@ -else -# Run make locally - @$(MAKE) --warn-undefined-variables -f tools/make/common.mk $@ -endif +# Have everything-else ("%") depend on _run (which uses +# $(MAKECMDGOALS) to decide what to run), rather than having +# everything else run $(MAKE) directly, since that'd end up running +# multiple sub-Makes if you give multiple targets on the CLI. +_run: + $(MAKE) --warn-undefined-variables -f tools/make/common.mk $(MAKECMDGOALS) +.PHONY: _run +$(if $(MAKECMDGOALS),$(MAKECMDGOALS): %: _run) diff --git a/tools/docker/builder/Dockerfile b/tools/docker/builder/Dockerfile deleted file mode 100644 index 522937f84e..0000000000 --- a/tools/docker/builder/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Make sure the tooling versions are same as defined in the -# CI https://github.com/envoyproxy/gateway/blob/main/.github/workflows/build_and_test.yaml -# as well as in the Developer docs - https://github.com/envoyproxy/gateway/blob/main/DEVELOPER.md#prerequisites - -# go -FROM golang:1.18.2 as builder - -ENV YAMLLINT_VERSION=1.24.2 -ENV GOLINT_VERSION=v1.46.2 -ENV CODESPELL_VERSION=v2.1.0 - -# docker CLI -RUN curl -fsSL https://get.docker.com | VERSION=20.10.16 sh - -# python -RUN apt-get update && apt-get install -y --no-install-recommends python3-pip - -# golangci Lint -RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s ${GOLINT_VERSION} - -# pip install -RUN python3 -m pip install --no-cache-dir yamllint==${YAMLLINT_VERSION} -RUN python3 -m pip install --no-cache-dir codespell==${CODESPELL_VERSION} - -WORKDIR /workspace -ENTRYPOINT ["make", "--warn-undefined-variables"]