From b2c411fd2a86bb51644c01e1aa6d8191df9442fb Mon Sep 17 00:00:00 2001 From: Dustin Brickwood Date: Tue, 12 Apr 2022 22:13:08 -0500 Subject: [PATCH 1/5] chore: adds docker ci --- .github/publish-docker.yml | 99 ++++++++++++++++++++++ docker/Parachain.Dockerfile | 14 +-- docker/Standalone.Dockerfile | 38 +++++++++ scripts/build-parachain-docker.sh | 14 +++ scripts/build-protocol-substrate-docker.sh | 24 ++++++ scripts/build-standalone-docker.sh | 24 ++++++ 6 files changed, 206 insertions(+), 7 deletions(-) create mode 100644 .github/publish-docker.yml create mode 100644 docker/Standalone.Dockerfile create mode 100644 scripts/build-parachain-docker.sh create mode 100644 scripts/build-protocol-substrate-docker.sh create mode 100644 scripts/build-standalone-docker.sh diff --git a/.github/publish-docker.yml b/.github/publish-docker.yml new file mode 100644 index 000000000..0e19f9cd2 --- /dev/null +++ b/.github/publish-docker.yml @@ -0,0 +1,99 @@ +name: Container Image Release + +on: + push: + # Publish `master` as Container `edge` image. + branches: + - master + # Publish `v1.2.3` tags as releases. + tags: + - v* + +jobs: + substrate-protocol-push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v2 + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + + - uses: actions/checkout@v2 + - name: Build Image + run: ./scripts/build-standalone-docker.sh + + - name: Install docker-slim + run: curl -sL https://raw.githubusercontent.com/docker-slim/docker-slim/master/scripts/install-dockerslim.sh | sudo -E bash - + + - name: Slim down the Image + run: docker-slim build --target webb-tools/protocol-substrate-standalone-node --http-probe-off --entrypoint webb-standalone-node + + - name: Log into registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/protocol-substrate-standalone-node + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + # Use Docker `edge` tag convention + [ "$VERSION" == "main" ] && VERSION=edge + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag webb-tools/protocol-substrate-standalone-node.slim $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + + dkg-standalone-push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + permissions: + packages: write + contents: read + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v2 + + - name: Build Image + run: ./scripts/build-standalone-docker.sh + + - name: Install docker-slim + run: curl -sL https://raw.githubusercontent.com/docker-slim/docker-slim/master/scripts/install-dockerslim.sh | sudo -E bash - + + - name: Slim down the Image + run: docker-slim build --target webb-tools/dkg-standalone-node --http-probe-off --entrypoint dkg-standalone-node + + - name: Log into registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/dkg-standalone-node + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `edge` tag convention + [ "$VERSION" == "master" ] && VERSION=edge + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag webb-tools/dkg-standalone-node.slim $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION diff --git a/docker/Parachain.Dockerfile b/docker/Parachain.Dockerfile index faf2ea2e3..a08e47374 100644 --- a/docker/Parachain.Dockerfile +++ b/docker/Parachain.Dockerfile @@ -1,8 +1,8 @@ FROM rust:buster as builder WORKDIR /app -RUN rustup default nightly-2021-11-07 && \ - rustup target add wasm32-unknown-unknown --toolchain nightly-2021-11-07 +RUN rustup default nightly-2022-02-01 && \ + rustup target add wasm32-unknown-unknown --toolchain nightly-2022-02-01 # Install Required Packages RUN apt-get update && apt-get install -y git clang curl libssl-dev llvm libudev-dev libgmp3-dev && rm -rf /var/lib/apt/lists/* @@ -13,7 +13,7 @@ ARG BUILD_ARGS COPY . . # Build DKG Parachain Node -RUN cargo build --release -p dkg-node +RUN cargo build --release -p egg-collator # ============= @@ -21,11 +21,11 @@ FROM phusion/baseimage:bionic-1.0.0 RUN useradd -m -u 1000 -U -s /bin/sh -d /dkg dkg -COPY --from=builder /app/target/release/dkg-node /usr/local/bin +COPY --from=builder /app/target/release/egg-collator /usr/local/bin # checks -RUN ldd /usr/local/bin/dkg-node && \ - /usr/local/bin/dkg-node --version +RUN ldd /usr/local/bin/egg-collator && \ + /usr/local/bin/egg-collator --version # Shrinking RUN rm -rf /usr/lib/python* && \ @@ -39,4 +39,4 @@ RUN chown -R dkg:dkg /dkg/data VOLUME ["/dkg/data"] -ENTRYPOINT [ "/usr/local/bin/dkg-node" ] \ No newline at end of file +ENTRYPOINT [ "/usr/local/bin/egg-collator" ] \ No newline at end of file diff --git a/docker/Standalone.Dockerfile b/docker/Standalone.Dockerfile new file mode 100644 index 000000000..48c529b2d --- /dev/null +++ b/docker/Standalone.Dockerfile @@ -0,0 +1,38 @@ +FROM rust:buster as builder +WORKDIR /app + +RUN rustup default nightly-2022-02-01 && \ + rustup target add wasm32-unknown-unknown --toolchain nightly-2022-02-01 + +# Install Required Packages +RUN apt-get update && apt-get install -y git clang curl libssl-dev llvm libudev-dev libgmp3-dev && rm -rf /var/lib/apt/lists/* + +ARG GIT_COMMIT= +ENV GIT_COMMIT=$GIT_COMMIT +ARG BUILD_ARGS + +COPY . . +# Build Standalone Node. +RUN git submodule update --init && \ + cargo build --release -p egg-standalone-node + + +# =============== + +FROM phusion/baseimage:bionic-1.0.0 + +COPY --from=builder /app/target/release/egg-standalone-node /usr/local/bin + +RUN useradd -m -u 1000 -U -s /bin/sh -d /app webb && \ + mkdir -p /data /app/.local/share/webb && \ + chown -R webb:webb /data && \ + ln -s /data /app/.local/share/webb && \ + # Sanity checks + ldd /usr/local/bin/egg-standalone-node && \ + /usr/local/bin/egg-standalone-node --version + +USER webb +EXPOSE 30333 9933 9944 9615 33334 +VOLUME ["/data"] + +ENTRYPOINT [ "/usr/local/bin/egg-standalone-node"] \ No newline at end of file diff --git a/scripts/build-parachain-docker.sh b/scripts/build-parachain-docker.sh new file mode 100644 index 000000000..e17cfaa86 --- /dev/null +++ b/scripts/build-parachain-docker.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e + +# The following line ensure we run from the project root +PROJECT_ROOT=`git rev-parse --show-toplevel` +cd $PROJECT_ROOT + +VERSION=`grep "^version" ./node/Cargo.toml | egrep -o "([0-9\.]+)"` +NODE_NAME=ghcr.io/webb-tools/dkg-node +BUILD_ARGS="cargo build --release -p dkg-node" + +docker build -f ./docker/Parachain.Dockerfile . -t $NODE_NAME:$VERSION --build-arg GIT_COMMIT=${VERSION} --build-arg BUILD_ARGS="$BUILD_ARGS" +docker push $NODE_NAME:$VERSION \ No newline at end of file diff --git a/scripts/build-protocol-substrate-docker.sh b/scripts/build-protocol-substrate-docker.sh new file mode 100644 index 000000000..6eabb96c3 --- /dev/null +++ b/scripts/build-protocol-substrate-docker.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +pushd . + +# The following line ensure we run from the project root +PROJECT_ROOT=`git rev-parse --show-toplevel` +cd $PROJECT_ROOT + +# Find the current version from Cargo.toml +VERSION=`grep "^version" ./standalone/node/Cargo.toml | egrep -o "([0-9\.]+)"` +GITUSER=webb-tools +IMAGE_NAME=protocol-substrate-standalone-node + +# Build the image +echo "Building ${GITUSER}/${IMAGE_NAME}:latest docker image, hang on!" +time docker build -f ./docker/Standalone.Dockerfile -t ${GITUSER}/${IMAGE_NAME}:latest . +docker tag ${GITUSER}/${IMAGE_NAME}:latest ${GITUSER}/${IMAGE_NAME}:v${VERSION} + +# Show the list of available images for this repo +echo "Image is ready" +docker images | grep ${IMAGE_NAME} + +popd diff --git a/scripts/build-standalone-docker.sh b/scripts/build-standalone-docker.sh new file mode 100644 index 000000000..8f90521ad --- /dev/null +++ b/scripts/build-standalone-docker.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +pushd . + +# The following line ensure we run from the project root +PROJECT_ROOT=`git rev-parse --show-toplevel` +cd $PROJECT_ROOT + +# Find the current version from Cargo.toml +VERSION=`grep "^version" ./standalone/node/Cargo.toml | egrep -o "([0-9\.]+)"` +GITUSER=webb-tools +IMAGE_NAME=egg-standalone-node + +# Build the image +echo "Building ${GITUSER}/${IMAGE_NAME}:latest docker image, hang on!" +time docker build -f ./docker/Standalone.Dockerfile -t ${GITUSER}/${IMAGE_NAME}:latest . +docker tag ${GITUSER}/${IMAGE_NAME}:latest ${GITUSER}/${IMAGE_NAME}:v${VERSION} + +# Show the list of available images for this repo +echo "Image is ready" +docker images | grep ${IMAGE_NAME} + +popd From 72153e05cdbe5598363fa643a2e4e548bd75f360 Mon Sep 17 00:00:00 2001 From: Dustin Brickwood Date: Wed, 1 Jun 2022 14:48:58 -0500 Subject: [PATCH 2/5] chore: update workflow for docker build and push --- .github/publish-docker.yml | 32 +++++++++++++++++-------------- docker/Parachain.Dockerfile | 6 +++--- docker/Standalone.Dockerfile | 12 ++++++------ scripts/build-parachain-docker.sh | 2 +- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/.github/publish-docker.yml b/.github/publish-docker.yml index 0e19f9cd2..093a9c72f 100644 --- a/.github/publish-docker.yml +++ b/.github/publish-docker.yml @@ -1,4 +1,4 @@ -name: Container Image Release +name: build-push on: push: @@ -10,20 +10,19 @@ on: - v* jobs: - substrate-protocol-push: + egg-standalone-node: runs-on: ubuntu-latest if: github.event_name == 'push' permissions: packages: write contents: read steps: - - uses: actions/checkout@v2 - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.9.1 with: access_token: ${{ github.token }} - - uses: actions/checkout@v2 + - name: Build Image run: ./scripts/build-standalone-docker.sh @@ -31,28 +30,33 @@ jobs: run: curl -sL https://raw.githubusercontent.com/docker-slim/docker-slim/master/scripts/install-dockerslim.sh | sudo -E bash - - name: Slim down the Image - run: docker-slim build --target webb-tools/protocol-substrate-standalone-node --http-probe-off --entrypoint webb-standalone-node + run: docker-slim build --target webb-tools/egg-standalone-node --http-probe-off --entrypoint egg-standalone-node - name: Log into registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - + - name: Push image run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/protocol-substrate-standalone-node + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/egg-standalone-node + # Change all uppercase to lowercase IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + # Strip git ref prefix from version VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + # Strip "v" prefix from tag name [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + # Use Docker `edge` tag convention [ "$VERSION" == "main" ] && VERSION=edge + echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION - docker tag webb-tools/protocol-substrate-standalone-node.slim $IMAGE_ID:$VERSION + docker tag webb-tools/egg-standalone-node.slim $IMAGE_ID:$VERSION docker push $IMAGE_ID:$VERSION - dkg-standalone-push: + egg-parachain-node: runs-on: ubuntu-latest if: github.event_name == 'push' permissions: @@ -66,20 +70,20 @@ jobs: - uses: actions/checkout@v2 - name: Build Image - run: ./scripts/build-standalone-docker.sh + run: ./scripts/build-parachain-docker.sh - name: Install docker-slim run: curl -sL https://raw.githubusercontent.com/docker-slim/docker-slim/master/scripts/install-dockerslim.sh | sudo -E bash - - name: Slim down the Image - run: docker-slim build --target webb-tools/dkg-standalone-node --http-probe-off --entrypoint dkg-standalone-node + run: docker-slim build --target webb-tools/egg-standalone-node --http-probe-off --entrypoint egg-collator - name: Log into registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: Push image run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/dkg-standalone-node + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/egg-collator # Change all uppercase to lowercase IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') @@ -91,9 +95,9 @@ jobs: [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') # Use Docker `edge` tag convention - [ "$VERSION" == "master" ] && VERSION=edge + [ "$VERSION" == "main" ] && VERSION=edge echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION - docker tag webb-tools/dkg-standalone-node.slim $IMAGE_ID:$VERSION + docker tag webb-tools/egg-collator.slim $IMAGE_ID:$VERSION docker push $IMAGE_ID:$VERSION diff --git a/docker/Parachain.Dockerfile b/docker/Parachain.Dockerfile index a08e47374..2e5684117 100644 --- a/docker/Parachain.Dockerfile +++ b/docker/Parachain.Dockerfile @@ -1,5 +1,5 @@ FROM rust:buster as builder -WORKDIR /app +WORKDIR /network RUN rustup default nightly-2022-02-01 && \ rustup target add wasm32-unknown-unknown --toolchain nightly-2022-02-01 @@ -13,7 +13,7 @@ ARG BUILD_ARGS COPY . . # Build DKG Parachain Node -RUN cargo build --release -p egg-collator +RUN cargo build --release --locked -p egg-collator # ============= @@ -21,7 +21,7 @@ FROM phusion/baseimage:bionic-1.0.0 RUN useradd -m -u 1000 -U -s /bin/sh -d /dkg dkg -COPY --from=builder /app/target/release/egg-collator /usr/local/bin +COPY --from=builder /network/target/release/egg-collator /usr/local/bin # checks RUN ldd /usr/local/bin/egg-collator && \ diff --git a/docker/Standalone.Dockerfile b/docker/Standalone.Dockerfile index 48c529b2d..817a7e43f 100644 --- a/docker/Standalone.Dockerfile +++ b/docker/Standalone.Dockerfile @@ -1,5 +1,5 @@ FROM rust:buster as builder -WORKDIR /app +WORKDIR /network RUN rustup default nightly-2022-02-01 && \ rustup target add wasm32-unknown-unknown --toolchain nightly-2022-02-01 @@ -14,19 +14,19 @@ ARG BUILD_ARGS COPY . . # Build Standalone Node. RUN git submodule update --init && \ - cargo build --release -p egg-standalone-node + cargo build --release --locked -p egg-standalone-node # =============== FROM phusion/baseimage:bionic-1.0.0 -COPY --from=builder /app/target/release/egg-standalone-node /usr/local/bin +COPY --from=builder /network/target/release/egg-standalone-node /usr/local/bin -RUN useradd -m -u 1000 -U -s /bin/sh -d /app webb && \ - mkdir -p /data /app/.local/share/webb && \ +RUN useradd -m -u 1000 -U -s /bin/sh -d /network webb && \ + mkdir -p /data /network/.local/share/webb && \ chown -R webb:webb /data && \ - ln -s /data /app/.local/share/webb && \ + ln -s /data /network/.local/share/webb && \ # Sanity checks ldd /usr/local/bin/egg-standalone-node && \ /usr/local/bin/egg-standalone-node --version diff --git a/scripts/build-parachain-docker.sh b/scripts/build-parachain-docker.sh index e17cfaa86..d0fa0fca6 100644 --- a/scripts/build-parachain-docker.sh +++ b/scripts/build-parachain-docker.sh @@ -7,7 +7,7 @@ PROJECT_ROOT=`git rev-parse --show-toplevel` cd $PROJECT_ROOT VERSION=`grep "^version" ./node/Cargo.toml | egrep -o "([0-9\.]+)"` -NODE_NAME=ghcr.io/webb-tools/dkg-node +NODE_NAME=ghcr.io/webb-tools/egg-collator BUILD_ARGS="cargo build --release -p dkg-node" docker build -f ./docker/Parachain.Dockerfile . -t $NODE_NAME:$VERSION --build-arg GIT_COMMIT=${VERSION} --build-arg BUILD_ARGS="$BUILD_ARGS" From a68878e9ca75066e158aa46d339d79056098dc43 Mon Sep 17 00:00:00 2001 From: Dustin Brickwood Date: Wed, 1 Jun 2022 14:52:39 -0500 Subject: [PATCH 3/5] chore: add workflow dispatch to action --- .github/publish-docker.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/publish-docker.yml b/.github/publish-docker.yml index 093a9c72f..b81326f11 100644 --- a/.github/publish-docker.yml +++ b/.github/publish-docker.yml @@ -1,18 +1,18 @@ name: build-push - + on: push: - # Publish `master` as Container `edge` image. - branches: - - master - # Publish `v1.2.3` tags as releases. - tags: + branches: [ main ] + pull_request: + branches: [ main ] + tags: - v* + workflow_dispatch: + jobs: egg-standalone-node: runs-on: ubuntu-latest - if: github.event_name == 'push' permissions: packages: write contents: read @@ -58,7 +58,6 @@ jobs: egg-parachain-node: runs-on: ubuntu-latest - if: github.event_name == 'push' permissions: packages: write contents: read From 4d44c925217161c96cb43842acfdf7eb09f679bf Mon Sep 17 00:00:00 2001 From: Dustin Brickwood Date: Wed, 1 Jun 2022 14:57:46 -0500 Subject: [PATCH 4/5] chore: add workflow dispatch to action --- .github/publish-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/publish-docker.yml b/.github/publish-docker.yml index b81326f11..e045ca64d 100644 --- a/.github/publish-docker.yml +++ b/.github/publish-docker.yml @@ -1,5 +1,5 @@ name: build-push - + on: push: branches: [ main ] From 98b5181c39e82b7111f9393dda5ddb109c7ab1fc Mon Sep 17 00:00:00 2001 From: Dustin Brickwood Date: Wed, 1 Jun 2022 15:00:06 -0500 Subject: [PATCH 5/5] chore: test action --- .github/publish-docker.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/publish-docker.yml b/.github/publish-docker.yml index e045ca64d..a7eaa880a 100644 --- a/.github/publish-docker.yml +++ b/.github/publish-docker.yml @@ -11,6 +11,12 @@ on: workflow_dispatch: jobs: + dummy: + runs-on: ubuntu-latest + steps: + - name: dummy + run: | + echo "Dummy skip step" egg-standalone-node: runs-on: ubuntu-latest permissions: