diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5bc20d744c5e..7cdc926818ae 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -52,16 +52,15 @@ Dependencies: - [runc](https://github.com/opencontainers/runc) - [containerd](https://github.com/containerd/containerd) (if you want to use containerd worker) - -The following command installs `buildkitd` and `buildctl` to `/usr/local/bin`: - ```bash -make && sudo make install -``` +# installs buildkitd and buildctl to /usr/local/bin +$ make && sudo make install -To build containerized `moby/buildkit:local` and `moby/buildkit:local-rootless` images: -```bash -make images +# build moby/buildkit:local and moby/buildkit:local-rootless images +$ make images + +# build docker/dockerfile:local and docker/dockerfile:local-labs images +$ make frontends ``` ### Run the unit- and integration-tests @@ -94,13 +93,15 @@ TESTPKGS=./client TESTFLAGS="--run //worker=containerd -v" ./hack/test integrati DOCKERFILE_RELEASES=labs TESTFLAGS="--run /TestRunGlobalNetwork/worker=oci$/ -v" ./hack/test dockerfile ``` -Set `TEST_KEEP_CACHE=1` for the test framework to keep external dependant images in a docker volume -if you are repeatedly calling `./hack/test` script. This helps to avoid rate limiting on the remote registry side. +> **Note** +> +> Set `TEST_KEEP_CACHE=1` for the test framework to keep external dependant +> images in a docker volume if you are repeatedly calling `./hack/test` script. +> This helps to avoid rate limiting on the remote registry side. Updating vendored dependencies: ```bash -# update vendor.conf make vendor ``` @@ -110,8 +111,6 @@ Validating your updates before submission: make validate-all ``` - - ### Pull requests are always welcome Not sure if that typo is worth a pull request? Found a bug and know how to fix diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57512e8e48fa..a2fb30330736 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,18 +6,14 @@ concurrency: on: schedule: - - cron: '0 10 * * *' # everyday at 10am + - cron: '0 10 * * *' workflow_dispatch: push: branches: - 'master' tags: - 'v*' - - 'dockerfile/*' pull_request: - branches: - - 'master' - - 'v*' paths-ignore: - 'README.md' - 'docs/**' @@ -26,25 +22,20 @@ on: env: REPO_SLUG_ORIGIN: "moby/buildkit:latest" REPO_SLUG_TARGET: "moby/buildkit" - DF_REPO_SLUG_TARGET: "docker/dockerfile-upstream" - PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64" - CACHE_GHA_SCOPE_IT: "integration-tests" - CACHE_GHA_SCOPE_BINARIES: "binaries" + CACHE_GHA_SCOPE_IT: "build-integration-tests" CACHE_GHA_SCOPE_CROSS: "cross" TESTFLAGS: "-v --parallel=6 --timeout=30m" BUILDX_VERSION: "v0.9.1" # leave empty to use the one available on GitHub virtual environment GO_VERSION: "1.19" + DESTDIR: "./bin" jobs: - base: + prepare-test: runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -56,29 +47,23 @@ jobs: driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - name: Build ${{ env.CACHE_GHA_SCOPE_BINARIES }} - run: | - ./hack/build_ci_first_pass binaries - env: - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} - CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} - - - name: Build ${{ env.CACHE_GHA_SCOPE_IT }} - run: | - ./hack/build_ci_first_pass integration-tests - env: - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} - CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} + name: Build + uses: docker/bake-action@v2 + with: + targets: integration-tests-base + set: | + *.cache-from=type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} + *.cache-to=type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} test: runs-on: ubuntu-20.04 - needs: [base] + needs: + - prepare-test strategy: fail-fast: false matrix: pkg: - ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend - - ./frontend/dockerfile worker: - containerd - containerd-rootless @@ -89,10 +74,6 @@ jobs: - oci-snapshotter-stargz typ: - integration - - dockerfile - exclude: - - pkg: ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend - typ: dockerfile include: - pkg: ./... skip-integration-tests: 1 @@ -126,7 +107,7 @@ jobs: TEST_COVERAGE: 1 TESTPKGS: ${{ matrix.pkg }} SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} + CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} - name: Upload coverage file uses: actions/upload-artifact@v3 @@ -136,7 +117,8 @@ jobs: test-nydus: runs-on: ubuntu-20.04 - needs: [base] + needs: + - prepare-test strategy: fail-fast: false matrix: @@ -187,7 +169,7 @@ jobs: test-s3: runs-on: ubuntu-20.04 needs: - - base + - prepare-test steps: - name: Checkout @@ -210,7 +192,7 @@ jobs: test-azblob: runs-on: ubuntu-20.04 needs: - - base + - prepare-test steps: - name: Checkout @@ -305,9 +287,8 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 + with: + fetch-depth: 0 - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -319,55 +300,56 @@ jobs: driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - name: Cross - run: | - ./hack/cross - env: - PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 - RUNC_PLATFORMS: ${{ env.PLATFORMS }} - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} - CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} - - release-base: - runs-on: ubuntu-20.04 - outputs: - tag: ${{ steps.prep.outputs.tag }} - push: ${{ steps.prep.outputs.push }} - steps: - - name: Prepare - id: prep - run: | - TAG=pr - PUSH=false - if [ "${{ github.event_name }}" = "schedule" ]; then - TAG=nightly - PUSH=push - elif [[ $GITHUB_REF == refs/tags/v* ]]; then - TAG=${GITHUB_REF#refs/tags/} - PUSH=push - elif [[ $GITHUB_REF == refs/heads/* ]]; then - TAG=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - PUSH=push - fi - echo "tag=${TAG}" >>${GITHUB_OUTPUT} - echo "push=${PUSH}" >>${GITHUB_OUTPUT} + name: Build + uses: docker/bake-action@v2 + with: + targets: binaries-cross + set: | + *.cache-from=type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} + *.cache-to=type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} + *.output=type=cacheonly image: runs-on: ubuntu-20.04 - needs: [release-base, test, cross] + needs: + - cross + - test strategy: fail-fast: false matrix: - target-stage: + target: - '' - rootless steps: - name: Checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 + name: Prepare + run: | + TAGS=pr + TAG_SUFFIX= + PUSH=false + if [ -n "${{ matrix.target }}" ]; then + TAG_SUFFIX=-${{ matrix.target }} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + TAGS=nightly${TAG_SUFFIX} + PUSH=true + elif [[ $GITHUB_REF == refs/tags/v* ]]; then + TAGS="$(./hack/tags ${{ matrix.target }})" + PUSH=true + elif [[ $GITHUB_REF == refs/heads/* ]]; then + TAGS=${GITHUB_REF#refs/heads/}${TAG_SUFFIX} + PUSH=true + fi + echo "TAGS<> $GITHUB_ENV + echo "$TAGS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "TAG_SUFFIX=${TAG_SUFFIX}" >> $GITHUB_ENV + echo "PUSH=${PUSH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -379,31 +361,75 @@ jobs: driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - name: Login to DockerHub - if: needs.release-base.outputs.push == 'push' + name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REPO_SLUG_TARGET }} + tags: | + ${{ env.TAGS }} + bake-target: meta-helper + - + name: Login to Docker Hub + if: env.PUSH == 'true' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build ${{ needs.release-base.outputs.tag }} - run: | - ./hack/images "${{ needs.release-base.outputs.tag }}" "$REPO_SLUG_TARGET" "${{ needs.release-base.outputs.push }}" + name: Build + uses: docker/bake-action@v2 + with: + files: | + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + targets: image-all + set: | + *.cache-from=type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} + *.cache-from=type=gha,scope=image${{ env.TAG_SUFFIX }} + *.cache-to=type=gha,scope=image${{ env.TAG_SUFFIX }} + *.output=type=image,buildinfo-attrs=true,push=${{ env.PUSH }} env: - TARGET: ${{ matrix.target-stage }} - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} type=gha,scope=image${{ matrix.target-stage }} - CACHE_TO: type=gha,scope=image${{ matrix.target-stage }} + IMAGE_TARGET: ${{ matrix.target }} + + prepare-binaries: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.platforms.outputs.matrix }} + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Create matrix + id: platforms + run: | + echo "matrix=$(docker buildx bake binaries-cross --print | jq -cr '.target."binaries-cross".platforms')" >> $GITHUB_OUTPUT + - + name: Show matrix + run: | + echo ${{ steps.platforms.outputs.matrix }} binaries: runs-on: ubuntu-20.04 - needs: [release-base, test, cross] + needs: + - prepare-binaries + strategy: + fail-fast: false + matrix: + platform: ${{ fromJson(needs.prepare-binaries.outputs.matrix) }} steps: + - + name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 + with: + fetch-depth: 0 - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -415,101 +441,47 @@ jobs: driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - name: Build ${{ needs.release-base.outputs.tag }} - run: | - ./hack/release-tar "${{ needs.release-base.outputs.tag }}" release-out - env: - PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} - - - name: Move artifacts - run: | - mv ./release-out/**/* ./release-out/ + name: Build + uses: docker/bake-action@v2 + with: + targets: release + set: | + *.platform=${{ matrix.platform }} + *.cache-from=type=gha,scope=binaries-${{ env.PLATFORM_PAIR }} + *.cache-to=type=gha,scope=binaries-${{ env.PLATFORM_PAIR }} - name: Upload artifacts uses: actions/upload-artifact@v3 with: name: buildkit - path: ./release-out/* + path: ${{ env.DESTDIR }}/* if-no-files-found: error - - - name: GitHub Release - if: startsWith(github.ref, 'refs/tags/v') - uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - draft: true - files: ./release-out/* - name: ${{ needs.release-base.outputs.tag }} - - frontend-base: - runs-on: ubuntu-20.04 - if: github.event_name != 'schedule' - outputs: - typ: ${{ steps.prep.outputs.typ }} - tag: ${{ steps.prep.outputs.tag }} - push: ${{ steps.prep.outputs.push }} - steps: - - - name: Prepare - id: prep - run: | - TYP=master - TAG=mainline - PUSH=false - if [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then - TYP=tag - TAG=${GITHUB_REF#refs/tags/} - PUSH=push - elif [[ $GITHUB_REF == refs/heads/* ]]; then - PUSH=push - fi - echo "typ=${TYP}" >>${GITHUB_OUTPUT} - echo "tag=${TAG}" >>${GITHUB_OUTPUT} - echo "push=${PUSH}" >>${GITHUB_OUTPUT} - frontend-image: - runs-on: ubuntu-20.04 - if: github.event_name != 'schedule' - needs: [frontend-base, test] + release: + runs-on: ubuntu-latest + needs: + - binaries + - test steps: - name: Checkout uses: actions/checkout@v3 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - version: ${{ env.BUILDX_VERSION }} - driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} - buildkitd-flags: --debug - - - name: Login to DockerHub - uses: docker/login-action@v2 - if: needs.frontend-base.outputs.push == 'push' + name: Download artifacts + uses: actions/download-artifact@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + name: buildkit + path: ${{ env.DESTDIR }} - - name: Build ${{ needs.frontend-base.outputs.typ }}/${{ needs.frontend-base.outputs.tag }} + name: List artifacts run: | - ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ needs.frontend-base.outputs.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" - env: - PLATFORMS: ${{ env.PLATFORMS }},linux/mips,linux/mipsle,linux/mips64,linux/mips64le - CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} - CACHE_TO: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} + tree -nh ${{ env.DESTDIR }} - - name: Build ${{ needs.frontend-base.outputs.typ }}/labs - if: needs.frontend-base.outputs.typ == 'master' - run: | - ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" labs "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" + name: GitHub Release + if: startsWith(github.ref, 'refs/tags/v') + uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 env: - PLATFORMS: ${{ env.PLATFORMS }},linux/mips,linux/mipsle,linux/mips64,linux/mips64le - CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + draft: true + files: ${{ env.DESTDIR }}/* diff --git a/.github/workflows/dockerd.yml b/.github/workflows/dockerd.yml index 16b8942ca241..a7c5f5e97639 100644 --- a/.github/workflows/dockerd.yml +++ b/.github/workflows/dockerd.yml @@ -12,7 +12,6 @@ on: env: REPO_SLUG_ORIGIN: "moby/buildkit:latest" CACHE_GHA_SCOPE_IT: "integration-tests" - CACHE_GHA_SCOPE_BINARIES: "binaries" TESTFLAGS: "-v --parallel=1 --timeout=30m" BUILDX_VERSION: "v0.9.1" # leave empty to use the one available on GitHub virtual environment @@ -140,4 +139,4 @@ jobs: TESTPKGS: "${{ matrix.pkg }}" TESTFLAGS: "${{ env.TESTFLAGS }} --run=//worker=dockerd$" SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" - CACHE_FROM: "type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}" + CACHE_FROM: "type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }}" diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 000000000000..efe4a9e2540b --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,236 @@ +name: frontend + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + push: + branches: + - 'master' + tags: + - 'dockerfile/*' + pull_request: + paths-ignore: + - 'README.md' + - 'docs/**' + - 'frontend/dockerfile/docs/**' + +env: + REPO_SLUG_ORIGIN: "moby/buildkit:latest" + REPO_SLUG_TARGET: "docker/dockerfile-upstream" + CACHE_GHA_SCOPE_IT: "frontend-integration-tests" + TESTFLAGS: "-v --parallel=6 --timeout=30m" + BUILDX_VERSION: "v0.9.1" # leave empty to use the one available on GitHub virtual environment + GO_VERSION: "1.19" + DESTDIR: "./bin" + +jobs: + prepare-test: + runs-on: ubuntu-20.04 + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + version: ${{ env.BUILDX_VERSION }} + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + buildkitd-flags: --debug + - + name: Build + uses: docker/bake-action@v2 + with: + targets: integration-tests-base + set: | + *.cache-from=type=gha,scope=${{ github.workflow }}-${{ env.CACHE_GHA_SCOPE_IT }} + *.cache-to=type=gha,scope=${{ github.workflow }}-${{ env.CACHE_GHA_SCOPE_IT }} + + test: + runs-on: ubuntu-20.04 + needs: + - prepare-test + strategy: + fail-fast: false + matrix: + pkg: + - ./frontend/dockerfile + worker: + - containerd + - containerd-rootless + - containerd-1.5 + - containerd-1.4 + - containerd-snapshotter-stargz + - oci + - oci-rootless + - oci-snapshotter-stargz + typ: + - integration + - dockerfile + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + version: ${{ env.BUILDX_VERSION }} + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + buildkitd-flags: --debug + - + name: Test pkg=${{ matrix.pkg }} ; typ=${{ matrix.typ }} ; worker=${{ matrix.worker }} + run: | + if [ -n "${{ matrix.worker }}" ]; then + export TESTFLAGS="${TESTFLAGS} --run=//worker=${{ matrix.worker }}$" + fi + ./hack/test ${{ matrix.typ }} + mv ./coverage/coverage.txt ./coverage/coverage-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.typ }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]').txt + env: + TEST_COVERAGE: 1 + TESTPKGS: ${{ matrix.pkg }} + CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} + - + name: Upload coverage file + uses: actions/upload-artifact@v3 + with: + name: coverage + path: ./coverage + + upload-coverage: + runs-on: ubuntu-20.04 + needs: + - test + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Download coverage files + uses: actions/download-artifact@v3 + with: + name: coverage + path: ./coverage + - + name: List coverage files + uses: actions/github-script@v6 + id: files + with: + result-encoding: string + script: | + return require('fs').readdirSync('./coverage', {withFileTypes: true}) + .filter(item => !item.isDirectory()) + .map(item => `./coverage/${item.name}`) + .join(','); + - + name: Send to Codecov + uses: codecov/codecov-action@v3 + with: + files: ${{ steps.files.outputs.result }} + + image: + runs-on: ubuntu-20.04 + needs: + - test + strategy: + fail-fast: false + matrix: + channel: + - mainline + - labs + steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - + name: Prepare + run: | + TAGS=pr + TAG_SUFFIX= + BUILD=true + PUSH=false + if [ "${{ matrix.channel }}" != "mainline" ]; then + TAG_SUFFIX=-${{ matrix.channel }} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + TAGS=nightly${TAG_SUFFIX} + PUSH=true + elif [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then + TAGS="$(./frontend/dockerfile/cmd/dockerfile-frontend/hack/tags ${GITHUB_REF#refs/tags/})" + PUSH=true + if [[ "${{ matrix.channel }}" != "mainline" ]] && [[ $GITHUB_REF != refs/tags/dockerfile/*-${{ matrix.channel }} ]]; then + BUILD=false + elif [[ "${{ matrix.channel }}" == "mainline" ]] && [[ $GITHUB_REF == refs/tags/dockerfile/*-${{ matrix.channel }} ]]; then + BUILD=false + fi + elif [[ $GITHUB_REF == refs/heads/* ]]; then + TAGS=${GITHUB_REF#refs/heads/}${TAG_SUFFIX} + PUSH=true + fi + echo "TAGS<> $GITHUB_ENV + echo "$TAGS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "TAG_SUFFIX=${TAG_SUFFIX}" >> $GITHUB_ENV + echo "BUILD=${BUILD}" >> $GITHUB_ENV + echo "PUSH=${PUSH}" >> $GITHUB_ENV + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + if: env.BUILD == 'true' + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + if: env.BUILD == 'true' + with: + version: ${{ env.BUILDX_VERSION }} + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + buildkitd-flags: --debug + - + name: Docker meta + id: meta + uses: docker/metadata-action@v4 + if: env.BUILD == 'true' + with: + images: | + ${{ env.REPO_SLUG_TARGET }} + tags: | + ${{ env.TAGS }} + bake-target: frontend-meta-helper + - + name: Login to Docker Hub + uses: docker/login-action@v2 + if: env.BUILD == 'true' && env.PUSH == 'true' + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build + uses: docker/bake-action@v2 + if: env.BUILD == 'true' + with: + files: | + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + targets: frontend-image-all + set: | + *.cache-from=type=gha,scope=frontend${{ env.TAG_SUFFIX }} + *.cache-to=type=gha,scope=frontend${{ env.TAG_SUFFIX }} + *.output=type=image,buildinfo-attrs=true,push=${{ env.PUSH }} + env: + FRONTEND_CHANNEL: ${{ matrix.channel }} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 0042f1c9ac2a..8ec147e9474f 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -9,13 +9,7 @@ on: push: branches: - 'master' - tags: - - 'v*' - - 'dockerfile/*' pull_request: - branches: - - 'master' - - 'v*' env: REPO_SLUG_ORIGIN: "moby/buildkit:latest" @@ -27,11 +21,11 @@ jobs: strategy: fail-fast: false matrix: - include: - - script: ./hack/lint - - script: ./hack/validate-vendor - - script: ./hack/validate-generated-files - - script: ./hack/validate-shfmt + target: + - lint + - validate-vendor + - validate-generated-files + - validate-shfmt steps: - name: Checkout @@ -46,4 +40,4 @@ jobs: - name: Run run: | - ${{ matrix.script }} + make ${{ matrix.target }} diff --git a/.gitignore b/.gitignore index 75c0a9be9885..bd9babd17443 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,5 @@ # please consider a global .gitignore https://help.github.com/articles/ignoring-files bin coverage -release-out .certs .tmp diff --git a/Dockerfile b/Dockerfile index b5f0d3c252a9..b4d66e19d3d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ -# syntax=docker/dockerfile-upstream:1.4 +# syntax=docker/dockerfile-upstream:master +ARG GO_VERSION=1.19 ARG RUNC_VERSION=v1.1.4 ARG CONTAINERD_VERSION=v1.6.10 # containerd v1.5 for integration tests @@ -22,7 +23,7 @@ RUN apk add --no-cache git # xx is a helper for cross-compilation FROM --platform=$BUILDPLATFORM tonistiigi/xx@sha256:1e96844fadaa2f9aea021b2b05299bc02fe4c39a92d8e735b93e8e2b15610128 AS xx -FROM --platform=$BUILDPLATFORM golang:1.19-alpine AS golatest +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS golatest # gobuild is base stage for compiling go/cgo FROM golatest AS gobuild-base @@ -90,11 +91,10 @@ RUN --mount=target=. --mount=target=/root/.cache,type=cache \ CGO_ENABLED=0 xx-go build -ldflags "$(cat /tmp/.ldflags) -extldflags '-static'" -tags "osusergo netgo static_build seccomp ${BUILDKITD_TAGS}" -o /usr/bin/buildkitd ./cmd/buildkitd && \ xx-verify --static /usr/bin/buildkitd -FROM scratch AS binaries-linux-helper +FROM scratch AS binaries-linux COPY --link --from=runc /usr/bin/runc /buildkit-runc # built from https://github.com/tonistiigi/binfmt/releases/tag/buildkit%2Fv7.0.0-29 COPY --link --from=tonistiigi/binfmt:buildkit-v7.0.0-29@sha256:5168f6c2b692b04a7c0102751220e293e109b3dc312eb22ee1a5547b42b58de6 / / -FROM binaries-linux-helper AS binaries-linux COPY --link --from=buildctl /usr/bin/buildctl / COPY --link --from=buildkitd /usr/bin/buildkitd / diff --git a/Makefile b/Makefile index 813fcdf0e2e0..4b5c5eb92bf1 100644 --- a/Makefile +++ b/Makefile @@ -1,46 +1,88 @@ +ifneq (, $(BUILDX_BIN)) + export BUILDX_CMD = $(BUILDX_BIN) +else ifneq (, $(shell docker buildx version)) + export BUILDX_CMD = docker buildx +else ifneq (, $(shell which buildx)) + export BUILDX_CMD = $(which buildx) +else + export BUILDX_CMD = docker buildx +endif + prefix=/usr/local bindir=$(prefix)/bin -binaries: FORCE - hack/binaries +.PHONY: binaries +binaries: + $(BUILDX_CMD) bake binaries + +.PHONY: cross +cross: + $(BUILDX_CMD) bake binaries-cross + +.PHONY: release +release: + $(BUILDX_CMD) bake release + mv -f $(CURDIR)/bin/release/**/* $(CURDIR)/bin/release/ + find $(CURDIR)/bin/release -type d -empty -delete -images: FORCE -# moby/buildkit:local and moby/buildkit:local-rootless are created on Docker - hack/images local moby/buildkit - TARGET=rootless hack/images local moby/buildkit +.PHONY: images +images: # moby/buildkit:local and moby/buildkit:local-rootless are created on Docker + $(BUILDX_CMD) bake image + IMAGE_TARGET=rootless $(BUILDX_CMD) bake image -install: FORCE +.PHONY: frontends +frontends: # docker/dockerfile:local and docker/dockerfile:local-labs are created on Docker + $(BUILDX_CMD) bake frontend-image + FRONTEND_CHANNEL=labs $(BUILDX_CMD) bake frontend-image + +.PHONY: install +install: mkdir -p $(DESTDIR)$(bindir) - install bin/* $(DESTDIR)$(bindir) + install bin/build/* $(DESTDIR)$(bindir) -clean: FORCE +.PHONY: clean +clean: rm -rf ./bin +.PHONY: test test: - ./hack/test integration gateway dockerfile + hack/test integration gateway dockerfile +.PHONY: lint lint: - ./hack/lint + $(BUILDX_CMD) bake lint +.PHONY: validate-vendor validate-vendor: - ./hack/validate-vendor + $(BUILDX_CMD) bake validate-vendor -validate-shfmt: - ./hack/validate-shfmt - -shfmt: - ./hack/shfmt +.PHONY: validate-authors +validate-authors: + $(BUILDX_CMD) bake validate-authors +.PHONY: validate-generated-files validate-generated-files: - ./hack/validate-generated-files + $(BUILDX_CMD) bake validate-generated-files + +.PHONY: validate-shfmt +validate-shfmt: + $(BUILDX_CMD) bake validate-shfmt -validate-all: test lint validate-vendor validate-generated-files +.PHONY: validate-all +validate-all: test lint validate-vendor validate-generated-files validate-shfmt +.PHONY: vendor vendor: - ./hack/update-vendor + hack/update-vendor +.PHONY: authors +authors: + $(BUILDX_CMD) bake authors + +.PHONY: generated-files generated-files: - ./hack/update-generated-files + $(BUILDX_CMD) bake generated-files -.PHONY: vendor generated-files test binaries images install clean lint validate-all validate-vendor validate-generated-files -FORCE: +.PHONY: shfmt +shfmt: + $(BUILDX_CMD) bake shfmt diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 000000000000..89d9821ce130 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,194 @@ +variable "GO_VERSION" { + default = "1.19" +} + +variable "BUILDKITD_TAGS" { + default = "" +} + +variable "IMAGE_TARGET" { + default = "" +} + +variable "FRONTEND_CHANNEL" { + default = "mainline" +} +variable "FRONTEND_BUILDTAGS" { + default = "" +} + +# Defines the output folder +variable "DESTDIR" { + default = "" +} +function "bindir" { + params = [defaultdir] + result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}" +} + +# Special target: https://github.com/docker/metadata-action#bake-definition +target "meta-helper" { + tags = [IMAGE_TARGET != "" ? "moby/buildkit:local-${IMAGE_TARGET}" : "moby/buildkit:local"] +} +target "frontend-meta-helper" { + tags = [FRONTEND_CHANNEL != "mainline" ? "docker/dockerfile:local-${FRONTEND_CHANNEL}" : "docker/dockerfile:local"] +} + +target "_common" { + args = { + GO_VERSION = GO_VERSION + BUILDKIT_CONTEXT_KEEP_GIT_DIR = 1 + } +} + +group "default" { + targets = ["binaries"] +} + +group "validate" { + targets = ["lint", "validate-vendor", "validate-generated-files", "validate-shfmt"] +} + +target "lint" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/lint.Dockerfile" + output = ["type=cacheonly"] +} + +target "validate-vendor" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/vendor.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + +target "validate-generated-files" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/generated-files.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + +target "validate-shfmt" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/shfmt.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + +target "validate-authors" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/authors.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + +target "vendor" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/vendor.Dockerfile" + target = "update" + output = ["."] +} + +target "generated-files" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/generated-files.Dockerfile" + target = "update" + output = ["."] +} + +target "shfmt" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/shfmt.Dockerfile" + target = "update" + output = ["."] +} + +target "authors" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/authors.Dockerfile" + target = "update" + output = ["."] +} + +target "binaries" { + inherits = ["_common"] + target = "binaries" + args = { + BUILDKITD_TAGS = BUILDKITD_TAGS + } + output = [bindir("build")] +} + +target "binaries-cross" { + inherits = ["binaries"] + output = [bindir("cross")] + platforms = [ + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm/v7", + "linux/arm64", + "linux/s390x", + "linux/ppc64le", + "linux/riscv64", + "windows/amd64", + "windows/arm64" + ] +} + +target "release" { + inherits = ["binaries-cross"] + target = "release" + output = [bindir("release")] +} + +target "image" { + inherits = ["_common", "meta-helper"] + target = IMAGE_TARGET + output = ["type=docker,buildinfo-attrs=true"] +} + +target "image-all" { + inherits = ["image"] + platforms = [ + "linux/amd64", + "linux/arm/v7", + "linux/arm64", + "linux/s390x", + "linux/ppc64le", + "linux/riscv64" + ] +} + +target "frontend-image" { + inherits = ["_common", "frontend-meta-helper"] + dockerfile = "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" + args = { + CHANNEL = FRONTEND_CHANNEL + BUILDTAGS = FRONTEND_BUILDTAGS + } + output = ["type=docker,buildinfo-attrs=true"] +} + +target "frontend-image-all" { + inherits = ["frontend-image"] + platforms = [ + "linux/amd64", + "linux/arm/v7", + "linux/arm64", + "linux/mips", + "linux/mipsle", + "linux/mips64", + "linux/mips64le", + "linux/s390x", + "linux/ppc64le", + "linux/riscv64" + ] +} + +target "integration-tests-base" { + inherits = ["_common"] + target = "integration-tests-base" + output = ["type=cacheonly"] +} diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index 23b1fd288fa0..dc7b7c8a0f04 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -1,9 +1,11 @@ # syntax=docker/dockerfile-upstream:master +ARG GO_VERSION="1.19" + # xx is a helper for cross-compilation FROM --platform=$BUILDPLATFORM tonistiigi/xx:master@sha256:d4254d9739ce2de9fb88e09bdc716aa0c65f0446a2a2143399f991d71136a3d4 AS xx -FROM --platform=$BUILDPLATFORM golang:1.19-alpine AS base +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS base RUN apk add git bash COPY --from=xx / / WORKDIR /src @@ -11,21 +13,32 @@ ENV GOFLAGS=-mod=vendor FROM base AS version ARG CHANNEL -# TODO: PKG should be inferred from go modules -RUN --mount=target=. \ - PKG=github.com/moby/buildkit/frontend/dockerfile/cmd/dockerfile-frontend VERSION=$(./frontend/dockerfile/cmd/dockerfile-frontend/hack/detect "$CHANNEL") REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) \ - && echo "-X main.Version=${VERSION} -X main.Revision=${REVISION} -X main.Package=${PKG}" | tee /tmp/.ldflags \ - && echo -n "${VERSION}" | tee /tmp/.version +ARG BUILDTAGS +RUN --mount=target=. < [push]" - exit 1 -} - -if [ $# != 4 ]; then - usage -fi - -parseTag() { - local prefix=$(echo $1 | cut -d/ -f 1) - if [[ "$prefix" != "dockerfile" ]]; then - echo "invalid tag $1" - exit 1 - fi - local suffix=$(echo $1 | awk -F- '{print $NF}') - local tagf=./frontend/dockerfile/release/$suffix/tags - if [ "$sufffix" == "$1" ] || [ ! -f $tagf ]; then - suffix="mainline" - fi - - local mainTag=$(echo $1 | cut -d/ -f 2) - - publishedNames=$REPO:$mainTag - - local versioned="" - # \d.\d.\d becomes latest - if [[ "$mainTag" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - publishedNames=${publishedNames},$REPO:latest - versioned=1 - fi - - # \d.\d.\d-channel becomes - if [[ "$mainTag" =~ ^[0-9]+\.[0-9]+\.[0-9]+-$suffix$ ]] && [ -f $tagf ]; then - publishedNames=${publishedNames},$REPO:$suffix - versioned=1 - fi - - # \d.\d.\d* -> \d.\d* -> \d* (except "0") - if [ "$versioned" == "1" ]; then - publishedNames=${publishedNames},$REPO:$(echo $mainTag | sed -E 's#^([0-9]+\.[0-9]+)\.[0-9]+#\1#') - if [ "$(echo $mainTag | sed -E 's#^([0-9]+)\.[0-9]+\.[0-9]+.*$#\1#')" != "0" ]; then - publishedNames=${publishedNames},$REPO:$(echo $mainTag | sed -E 's#^([0-9]+)\.[0-9]+\.[0-9]+#\1#') - fi - fi - - TAG=$suffix -} - -TYP=$1 -TAG=$2 -REPO=$3 -PUSH=$4 - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -pushFlag="push=false" -if [ "$PUSH" = "push" ]; then - pushFlag="push=true" -fi - -case $TYP in -"master") - tagf=./frontend/dockerfile/release/$TAG/tags - if [ ! -f $tagf ]; then - echo "invalid release $TAG" - exit 1 - fi - - buildTags=$(cat $tagf) - pushTag="master" - if [ "$TAG" != "mainline" ]; then - pushTag=${pushTag}-$TAG - fi - - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --output "type=image,name=$REPO:$pushTag,buildinfo-attrs=true,$pushFlag" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - ;; -"tag") - publishedNames="" - parseTag $TAG - tagf=./frontend/dockerfile/release/$TAG/tags - if [ ! -f $tagf ]; then - echo "no build tags found for $TAG" - exit 1 - fi - buildTags=$(cat $tagf) - - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --output "type=image,\"name=$publishedNames\",buildinfo-attrs=true,$pushFlag" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - ;; -"daily") - # if [ -z $DAILY_TARGETS ]; then - # DAILY_TARGETS="" - # fi - - for TAG in $DAILY_TARGETS; do - - tagf=./frontend/dockerfile/release/$TAG/tags - if [ ! -f $tagf ]; then - echo "invalid release $TAG" - exit 1 - fi - buildTags=$(cat $tagf) - - # find the buildID of the last pushed image - # returns a BuildID if rebuild needed - - tmp=$(mktemp -d -t buildid.XXXXXXXXXX) - dt=$(date +%Y%m%d) - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags \ - --target "buildid" \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --build-arg "REPO=$REPO" \ - --build-arg "DATE=$dt" \ - --output "type=local,dest=$tmp" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - - if [ -f $tmp/buildid ]; then - buildid=$(cat $tmp/buildid) - echo "buildid: $buildid" - - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --output "type=image,name=$REPO:$dt-$TAG,buildinfo-attrs=true,$pushFlag" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - rm $tmp/buildid - fi - rm -r $tmp - - done - - ;; -esac diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/tags b/frontend/dockerfile/cmd/dockerfile-frontend/hack/tags new file mode 100755 index 000000000000..300dd3c56c50 --- /dev/null +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/tags @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -e + +usage() { + echo "$0 version" + exit 1 +} + +VERSION=$1 +if [ -z "$VERSION" ]; then + usage +fi + +prefix=$(echo "$VERSION" | cut -d/ -f 1) +if [ "$prefix" != "dockerfile" ]; then + echo "invalid version $VERSION" + exit 1 +fi + +suffix=$(echo "$VERSION" | awk -F- '{print $NF}') +tagf=./frontend/dockerfile/release/$suffix/tags +if [ "$suffix" = "$VERSION" ] || [ ! -f "$tagf" ]; then + suffix="mainline" +fi + +mainTag=$(echo "$VERSION" | cut -d/ -f 2) +tags=${mainTag} +versioned="" + +# \d.\d.\d becomes latest +if [[ "$mainTag" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + tags="${tags}\nlatest" + versioned=1 +fi + +# \d.\d.\d-channel becomes +if [[ "$mainTag" =~ ^[0-9]+\.[0-9]+\.[0-9]+-$suffix$ ]] && [ -f "$tagf" ]; then + tags="${tags}\n$suffix" + versioned=1 +fi + +# \d.\d.\d* -> \d.\d* -> \d* (except "0") +if [ "$versioned" == "1" ]; then + tags="${tags}\n$(echo "$mainTag" | sed -E 's#^([0-9]+\.[0-9]+)\.[0-9]+#\1#')" + if [ "$(echo "$mainTag" | sed -E 's#^([0-9]+)\.[0-9]+\.[0-9]+.*$#\1#')" != "0" ]; then + tags="${tags}\n$(echo "$mainTag" | sed -E 's#^([0-9]+)\.[0-9]+\.[0-9]+#\1#')" + fi +fi + +echo -e "$tags" diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/version b/frontend/dockerfile/cmd/dockerfile-frontend/hack/version new file mode 100755 index 000000000000..26dbcc02a8d1 --- /dev/null +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/version @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +usage() { + echo "./version channel" + exit 1 +} + +if [ "$#" == 0 ]; then + usage +fi + +CHANNEL=$1 + +suffix= +if [ -n "$CHANNEL" ] && [ "$CHANNEL" != "mainline" ]; then + suffix=-$CHANNEL +fi + +version=$(git describe --always --tags --match "dockerfile/[0-9]*$suffix") +if [[ ! "$version" =~ "dockerfile" ]]; then + version=${version}$suffix +fi + +echo -n "$version" diff --git a/hack/binaries b/hack/binaries deleted file mode 100755 index cb4dacb2d679..000000000000 --- a/hack/binaries +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu - -: ${PLATFORMS=} -: ${BUILDKITD_TAGS=} - -buildkitdTagsFlags="" -if [ -n "$BUILDKITD_TAGS" ]; then - buildkitdTagsFlags="--build-arg=BUILDKITD_TAGS=\"$BUILDKITD_TAGS\"" -fi - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -buildxCmd build $platformFlag $buildkitdTagsFlags \ - --target "binaries" \ - --output "type=local,dest=./bin/" \ - . diff --git a/hack/build_ci_first_pass b/hack/build_ci_first_pass deleted file mode 100755 index 1d05c7e44812..000000000000 --- a/hack/build_ci_first_pass +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -TYP=$1 - -. $(dirname $0)/util -set -e - -usage() { - echo "usage: ./hack/build_ci_first_pass " - exit 1 -} - -if [ -z "$TYP" ]; then - usage -fi - -case $TYP in - "binaries") - buildxCmd build $cacheFromFlags $cacheToFlags \ - --target "binaries" \ - $currentcontext - ;; - "integration-tests") - buildxCmd build $cacheFromFlags $cacheToFlags \ - --target "integration-tests-base" \ - $currentcontext - ;; - *) - echo >&2 "Unknown type $TYP" - exit 1 - ;; -esac diff --git a/hack/cross b/hack/cross deleted file mode 100755 index 1502fd266d0e..000000000000 --- a/hack/cross +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -e - -: ${PLATFORMS=} - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -if [ -n "$RUNC_PLATFORMS" ]; then - buildxCmd build $cacheFromFlags $cacheToFlags \ - --target "binaries-linux-helper" \ - --platform "$RUNC_PLATFORMS" \ - $currentcontext -fi - -buildxCmd build $platformFlag $cacheFromFlags \ - $currentcontext diff --git a/hack/dockerfiles/authors.Dockerfile b/hack/dockerfiles/authors.Dockerfile new file mode 100644 index 000000000000..9dec80168b62 --- /dev/null +++ b/hack/dockerfiles/authors.Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile-upstream:master + +FROM alpine:3.16 AS gen +RUN apk add --no-cache git +WORKDIR /src +RUN --mount=type=bind,target=. < /out/AUTHORS + cat /out/AUTHORS +EOT + +FROM scratch AS update +COPY --from=gen /out / + +FROM gen AS validate +RUN --mount=type=bind,target=.,rw <&2 'ERROR: Authors result differs. Please update with "make authors"' + git status --porcelain -- AUTHORS + exit 1 + fi +EOT diff --git a/hack/dockerfiles/lint.Dockerfile b/hack/dockerfiles/lint.Dockerfile index 257a9e878196..f6566fb15ab0 100644 --- a/hack/dockerfiles/lint.Dockerfile +++ b/hack/dockerfiles/lint.Dockerfile @@ -1,6 +1,8 @@ # syntax=docker/dockerfile-upstream:master -FROM golang:1.19-alpine +ARG GO_VERSION=1.19 + +FROM golang:${GO_VERSION}-alpine ENV GOFLAGS="-buildvcs=false" RUN apk add --no-cache gcc musl-dev yamllint RUN wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.48.0 diff --git a/hack/dockerfiles/shfmt.Dockerfile b/hack/dockerfiles/shfmt.Dockerfile index fb5db637d90f..86b3e86847a6 100644 --- a/hack/dockerfiles/shfmt.Dockerfile +++ b/hack/dockerfiles/shfmt.Dockerfile @@ -1,4 +1,5 @@ # syntax=docker/dockerfile-upstream:master + FROM mvdan/shfmt:v3.1.2-alpine AS shfmt WORKDIR /src ARG SHFMT_FLAGS="-i 2 -ci" diff --git a/hack/dockerfiles/vendor.Dockerfile b/hack/dockerfiles/vendor.Dockerfile index 4108aaaecf1b..1228a97d5ca1 100644 --- a/hack/dockerfiles/vendor.Dockerfile +++ b/hack/dockerfiles/vendor.Dockerfile @@ -1,19 +1,37 @@ # syntax=docker/dockerfile-upstream:master -FROM golang:1.19-alpine AS vendored -RUN apk add --no-cache git +ARG GO_VERSION=1.19 + +FROM golang:${GO_VERSION}-alpine AS base +RUN apk add --no-cache git rsync WORKDIR /src -RUN --mount=target=/src,rw \ - --mount=target=/go/pkg/mod,type=cache \ - go mod tidy && go mod vendor && \ - mkdir /out && cp -r go.mod go.sum vendor /out + +FROM base AS vendored +RUN --mount=target=/context \ + --mount=target=.,type=tmpfs \ + --mount=target=/go/pkg/mod,type=cache <&2 'ERROR: Vendor result differs. Please vendor your package with "make vendor"' + git status --porcelain -- go.mod go.sum vendor + exit 1 + fi +EOT diff --git a/hack/generate-authors b/hack/generate-authors deleted file mode 100755 index 8acd3637c486..000000000000 --- a/hack/generate-authors +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -eu -o pipefail -x - -cd "$(dirname "$(readlink -f "$BASH_SOURCE")")/.." - -# see also ".mailmap" for how email addresses and names are deduplicated - -{ - cat <<-'EOH' - # This file lists all individuals having contributed content to the repository. - # For how it is generated, see `scripts/generate-authors.sh`. - EOH - echo - git log --format='%aN <%aE>' | LC_ALL=C.UTF-8 sort -uf -} >AUTHORS diff --git a/hack/images b/hack/images deleted file mode 100755 index 1b172059eef9..000000000000 --- a/hack/images +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -TAG=$1 -REPO=$2 -PUSH=$3 - -. $(dirname $0)/util -set -eu -o pipefail - -: ${PLATFORMS=} -: ${TARGET=} - -versionTag=$(git describe --always --tags --match "v[0-9]*") - -if [[ ! "$versionTag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - versionTag="" -fi - -usage() { - echo "usage: $0 [push]" - exit 1 -} - -if [ -z "$TAG" ] || [ -z "$REPO" ]; then - usage -fi - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -localmode="" -if [[ "$TAG" == "local" ]]; then - localmode="1" - if [ "$PUSH" = "push" ]; then - echo >&2 "local images cannot be pushed" - exit 1 - fi -fi - -outputFlag="--output=type=image,push=false" -if [ "$PUSH" = "push" ]; then - outputFlag="--output=type=image,buildinfo-attrs=true,push=true" -fi -if [ -n "$localmode" ]; then - outputFlag="--output=type=docker,buildinfo-attrs=true" -fi - -targetFlag="" -if [ -n "$TARGET" ]; then - targetFlag="--target=$TARGET" -fi - -tagNames="$REPO:$TAG" -if [ -n "$TARGET" ]; then - tagNames="$tagNames-$TARGET" -fi - -if [[ "$versionTag" == "$TAG" ]]; then - if [ -n "$TARGET" ]; then - tagNames="$tagNames $REPO:$TARGET" - else - tagNames="$tagNames $REPO:latest" - fi -fi - -importCacheFlags="" -for tagName in $tagNames; do - importCacheFlags="$importCacheFlags--cache-from=type=registry,ref=$tagName " -done -if [ -n "$cacheFromFlags" ]; then - importCacheFlags="$importCacheFlags$cacheFromFlags" -fi -if [ -n "$localmode" ]; then - importCacheFlags="" -fi - -exportCacheFlags="" -if [ -n "$cacheToFlags" ]; then - exportCacheFlags="$cacheToFlags" -elif [ "$PUSH" = "push" ]; then - exportCacheFlags="$exportCacheFlags--cache-to=type=inline " -fi - -tagFlags="" -for tagName in $tagNames; do - tagFlags="$tagFlags--tag=$tagName " -done - -buildxCmd build $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag \ - $currentcontext diff --git a/hack/lint b/hack/lint deleted file mode 100755 index 647e85b68269..000000000000 --- a/hack/lint +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -e - -buildxCmd build --file ./hack/dockerfiles/lint.Dockerfile . diff --git a/hack/release-tar b/hack/release-tar deleted file mode 100755 index bfad2d5336e7..000000000000 --- a/hack/release-tar +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -TAG=$1 -OUT=$2 - -. $(dirname $0)/util -set -eu -o pipefail - -: ${PLATFORMS=} - -usage() { - echo "usage: ./hack/release-tar " - exit 1 -} - -if [ -z "$TAG" ] || [ -z "$OUT" ]; then - usage -fi - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -buildxCmd build $platformFlag $cacheFromFlags \ - --target release \ - --output "type=local,dest=$OUT" \ - $currentcontext diff --git a/hack/shfmt b/hack/shfmt deleted file mode 100755 index 0368e680e5ee..000000000000 --- a/hack/shfmt +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -e - -buildxCmd build \ - --target "update" \ - --output "type=local,dest=." \ - --file "./hack/dockerfiles/shfmt.Dockerfile" \ - . diff --git a/hack/tags b/hack/tags new file mode 100755 index 000000000000..47ff60314aa2 --- /dev/null +++ b/hack/tags @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + +TARGET=$1 + +tag=$(git describe --always --tags --match "v[0-9]*") + +suffix= +if [ -n "$TARGET" ]; then + suffix=-${TARGET} +fi + +tags=${tag}${suffix} +if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [[ -z "$TARGET" ]]; then + tags="${tags}\nlatest" + else + tags="${tags}\n${TARGET}" + fi +fi + +echo -e "$tags" diff --git a/hack/update-generated-files b/hack/update-generated-files deleted file mode 100755 index d038d7d6b632..000000000000 --- a/hack/update-generated-files +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu - -output=$(mktemp -d -t buildctl-output.XXXXXXXXXX) - -buildxCmd build \ - --target "update" \ - --output "type=local,dest=$output" \ - --file "./hack/dockerfiles/generated-files.Dockerfile" \ - . - -cp -R "$output/." . -rm -rf $output diff --git a/hack/update-vendor b/hack/update-vendor index 18dd3237c608..f2a6a76ab933 100755 --- a/hack/update-vendor +++ b/hack/update-vendor @@ -1,16 +1,14 @@ #!/usr/bin/env bash -. $(dirname $0)/util set -eu +: "${BUILDX_CMD=docker buildx}" + output=$(mktemp -d -t buildx-output.XXXXXXXXXX) -buildxCmd build \ - --target "update" \ - --output "type=local,dest=$output" \ - --file "./hack/dockerfiles/vendor.Dockerfile" \ - . +set -x +${BUILDX_CMD} bake --set "*.output=$output" vendor rm -rf ./vendor cp -R "$output"/out/* . -rm -rf $output +rm -rf "$output" diff --git a/hack/validate-generated-files b/hack/validate-generated-files deleted file mode 100755 index 705ce5405ada..000000000000 --- a/hack/validate-generated-files +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu - -buildxCmd build \ - --target "validate" \ - --output "type=cacheonly" \ - --file "./hack/dockerfiles/generated-files.Dockerfile" \ - . diff --git a/hack/validate-shfmt b/hack/validate-shfmt deleted file mode 100755 index a1e33f0f9548..000000000000 --- a/hack/validate-shfmt +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env sh - -. $(dirname $0)/util -set -e - -buildxCmd build \ - --target validate \ - --output "type=cacheonly" \ - --file ./hack/dockerfiles/shfmt.Dockerfile \ - . diff --git a/hack/validate-vendor b/hack/validate-vendor deleted file mode 100755 index d60a6fe91ab9..000000000000 --- a/hack/validate-vendor +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case ${1:-} in - '') - . $(dirname $0)/util - buildxCmd build \ - --target validate \ - --file ./hack/dockerfiles/vendor.Dockerfile \ - . - ;; - check) - status="$(git status --porcelain -- go.mod go.sum vendor 2>/dev/null)" - diffs=$(echo "$status" | grep -v '^[RAD] ' || true) - if [ "$diffs" ]; then - { - set +x - echo 'The result of "make vendor" differs' - echo - echo "$diffs" - echo - echo 'Please vendor your package with "make vendor"' - echo - } >&2 - exit 1 - fi - echo 'Congratulations! All vendoring changes are done the right way.' - ;; -esac