diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index 5ddbecf3cec1..a141ab170870 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -26,6 +26,7 @@ env: SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" IMAGE_NAME: "moby/buildkit" PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64" + DESTDIR: "./bin" jobs: test: @@ -59,6 +60,9 @@ jobs: push: ${{ steps.prep.outputs.push }} platforms: ${{ steps.prep.outputs.platforms }} steps: + - + name: Checkout + uses: actions/checkout@v3 - name: Prepare id: prep @@ -79,10 +83,10 @@ jobs: fi echo "tag=${TAG}" >>${GITHUB_OUTPUT} echo "push=${PUSH}" >>${GITHUB_OUTPUT} - platforms=$(jq -c -n --argjson str '"${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64"' '$str|split(",")') + platforms=$(docker buildx bake release --print | jq -cr '.target."release".platforms') echo "platforms=$platforms" >>${GITHUB_OUTPUT} - cross: + binaries: runs-on: ubuntu-20.04 needs: - prepare @@ -99,6 +103,8 @@ jobs: - name: Checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Expose GitHub Runtime uses: crazy-max/ghaction-github-runtime@v2 @@ -115,18 +121,18 @@ jobs: - name: Build run: | - ./hack/release-tar "${{ needs.prepare.outputs.tag }}" release-out + make release env: RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} PLATFORMS: ${{ matrix.platform }} - CACHE_FROM: type=gha,scope=cross-${{ env.PLATFORM_PAIR }} - CACHE_TO: type=gha,scope=cross-${{ env.PLATFORM_PAIR }} + CACHE_FROM: type=gha,scope=binaries-${{ env.PLATFORM_PAIR }} + CACHE_TO: type=gha,scope=binaries-${{ env.PLATFORM_PAIR }} - name: Upload artifacts uses: actions/upload-artifact@v3 with: name: buildkit - path: ./release-out/* + path: ${{ env.DESTDIR }}/* if-no-files-found: error image: @@ -179,7 +185,7 @@ jobs: needs: - prepare - test - - cross + - binaries - image steps: - @@ -187,11 +193,11 @@ jobs: uses: actions/download-artifact@v3 with: name: buildkit - path: ./release-out/* + path: ${{ env.DESTDIR }} - name: List artifacts run: | - tree -nh ./release-out/ + tree -nh ${{ env.DESTDIR }} - name: GitHub Release if: startsWith(github.ref, 'refs/tags/v') @@ -200,5 +206,5 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: draft: true - files: ./release-out/* + files: ${{ env.DESTDIR }}/* name: ${{ needs.prepare.outputs.tag }} diff --git a/.gitignore b/.gitignore index 75c0a9be9885..bd9babd17443 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,5 @@ # please consider a global .gitignore https://help.github.com/articles/ignoring-files bin coverage -release-out .certs .tmp diff --git a/Makefile b/Makefile index d2697fbbf2fc..3830307cb5ac 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,11 @@ export BUILDX_CMD ?= docker buildx .PHONY: binaries binaries: - hack/binaries + $(BUILDX_CMD) bake binaries + +.PHONY: cross +cross: + $(BUILDX_CMD) bake binaries-cross .PHONY: images images: @@ -16,7 +20,11 @@ images: .PHONY: install install: mkdir -p $(DESTDIR)$(bindir) - install bin/* $(DESTDIR)$(bindir) + install bin/build/* $(DESTDIR)$(bindir) + +.PHONY: release +release: + ./hack/release .PHONY: clean clean: diff --git a/docker-bake.hcl b/docker-bake.hcl index 8d70ec83a273..97abaec3699a 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -10,6 +10,19 @@ variable "NODE_VERSION" { default = null } +variable "BUILDKITD_TAGS" { + default = null +} + +# Defines the output folder +variable "DESTDIR" { + default = "" +} +function "bindir" { + params = [defaultdir] + result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}" +} + target "_common" { args = { ALPINE_VERSION = ALPINE_VERSION @@ -19,6 +32,42 @@ target "_common" { } } +group "default" { + targets = ["binaries"] +} + +target "binaries" { + inherits = ["_common"] + target = "binaries" + args = { + BUILDKITD_TAGS = BUILDKITD_TAGS + } + output = [bindir("build")] +} + +target "binaries-cross" { + inherits = ["binaries"] + output = [bindir("cross")] + platforms = [ + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm/v7", + "linux/arm64", + "linux/s390x", + "linux/ppc64le", + "linux/riscv64", + "windows/amd64", + "windows/arm64" + ] +} + +target "release" { + inherits = ["binaries-cross"] + target = "release" + output = [bindir("release")] +} + group "validate" { targets = ["lint", "validate-vendor", "validate-doctoc", "validate-generated-files", "validate-shfmt"] } diff --git a/hack/binaries b/hack/binaries deleted file mode 100755 index cb4dacb2d679..000000000000 --- a/hack/binaries +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu - -: ${PLATFORMS=} -: ${BUILDKITD_TAGS=} - -buildkitdTagsFlags="" -if [ -n "$BUILDKITD_TAGS" ]; then - buildkitdTagsFlags="--build-arg=BUILDKITD_TAGS=\"$BUILDKITD_TAGS\"" -fi - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -buildxCmd build $platformFlag $buildkitdTagsFlags \ - --target "binaries" \ - --output "type=local,dest=./bin/" \ - . diff --git a/hack/cross b/hack/cross deleted file mode 100755 index b8be1ddcb7db..000000000000 --- a/hack/cross +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -e - -: ${PLATFORMS=} - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -buildxCmd build $platformFlag $cacheFromFlags \ - $currentcontext diff --git a/hack/release b/hack/release new file mode 100755 index 000000000000..20416b789e0a --- /dev/null +++ b/hack/release @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +: "${GITHUB_ACTIONS=}" +: "${GITHUB_REPOSITORY=}" +: "${GITHUB_RUN_ID=}" + +: "${BUILDX_CMD=docker buildx}" +: "${DESTDIR=./bin/release}" +: "${CACHE_FROM=}" +: "${CACHE_TO=}" +: "${RELEASE=false}" +: "${PLATFORMS=}" + +if [ -n "$CACHE_FROM" ]; then + for cfrom in $CACHE_FROM; do + setFlags+=(--set "*.cache-from=$cfrom") + done +fi +if [ -n "$CACHE_TO" ]; then + for cto in $CACHE_TO; do + setFlags+=(--set "*.cache-to=$cto") + done +fi +if [ -n "$PLATFORMS" ]; then + setFlags+=(--set "*.platform=$PLATFORMS") +fi +if ${BUILDX_CMD} build --help 2>&1 | grep -- '--attest' >/dev/null; then + prvattrs="mode=max" + if [ "$GITHUB_ACTIONS" = "true" ]; then + prvattrs="$prvattrs,builder-id=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + fi + setFlags+=(--set "*.attest=type=sbom") + setFlags+=(--set "*.attest=type=provenance,$prvattrs") +fi +if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then + setFlags+=(--set "*.no-cache-filter=git,gobuild-base") +fi + +output=$(mktemp -d -t buildkit-output.XXXXXXXXXX) + +( + set -x + ${BUILDX_CMD} bake "${setFlags[@]}" --set "*.args.BUILDKIT_MULTI_PLATFORM=true" --set "*.output=$output" release +) + +for pdir in "${output}"/*/; do + ( + cd "$pdir" + releasetar=$(find . -name '*.tar.gz') + filename=$(basename "${releasetar%.tar.gz}") + mv "provenance.json" "${filename}.provenance.json" + mv "sbom-binaries.spdx.json" "${filename}.sbom.json" + find . -name 'sbom*.json' -exec rm {} \; + ) +done + +mkdir -p "$DESTDIR" +mv "$output"/**/* "$DESTDIR/" +rm -rf "$output" diff --git a/hack/release-tar b/hack/release-tar deleted file mode 100755 index 308243f72096..000000000000 --- a/hack/release-tar +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -TAG=$1 -OUT=$2 - -. $(dirname $0)/util -set -eu -o pipefail - -: ${RELEASE=false} -: ${PLATFORMS=} - -usage() { - echo "usage: ./hack/release-tar " - exit 1 -} - -if [ -z "$TAG" ] || [ -z "$OUT" ]; then - usage -fi - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -nocacheFilterFlag="" -if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then - nocacheFilterFlag="--no-cache-filter=git,gobuild-base" -fi - -output=$(mktemp -d -t buildkit-output.XXXXXXXXXX) - -buildxCmd build $platformFlag $cacheFromFlags $nocacheFilterFlag $(buildAttestFlags) \ - --build-arg "BUILDKIT_MULTI_PLATFORM=true" \ - --target release \ - --output "type=local,dest=$output" \ - $currentcontext - -for pdir in "${output}"/*/; do - ( - cd "$pdir" - releasetar=$(find . -name '*.tar.gz') - filename=$(basename "${releasetar%.tar.gz}") - mv "provenance.json" "${filename}.provenance.json" - mv "sbom-binaries.spdx.json" "${filename}.sbom.json" - find . -name 'sbom*.json' -exec rm {} \; - ) -done - -mkdir -p "$OUT" -mv "$output"/**/* "$OUT/" -rm -rf $output