From a1aa87b09a50f96107b366a03f1cb5b8c04743da Mon Sep 17 00:00:00 2001 From: chevdor Date: Tue, 31 Jul 2018 14:55:57 +0200 Subject: [PATCH 1/4] Improve docker image size with a 2 stages image --- .dockerignore | 2 ++ README.adoc | 9 ++++---- ci/script.sh | 5 +++-- docker/Dockerfile | 41 +++++++++++++++------------------- docker/build.sh | 18 ++++++++++----- docker/cleanup.sh | 8 ------- docker/readme-docker.adoc | 2 +- docker/version | 13 ----------- scripts/build-demos.sh | 27 ++++++++++++++++++++++ build.sh => scripts/build.sh | 0 common.sh => scripts/common.sh | 3 +++ init.sh => scripts/init.sh | 2 +- 12 files changed, 73 insertions(+), 57 deletions(-) create mode 100644 .dockerignore delete mode 100755 docker/cleanup.sh delete mode 100755 docker/version create mode 100755 scripts/build-demos.sh rename build.sh => scripts/build.sh (100%) rename common.sh => scripts/common.sh (97%) rename init.sh => scripts/init.sh (86%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000..2b0e81eaf0a60 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +doc +target diff --git a/README.adoc b/README.adoc index 85ee539b1e675..943b0d0c68ac6 100644 --- a/README.adoc +++ b/README.adoc @@ -92,7 +92,7 @@ Then build the code: [source, shell] ---- -./build.sh # Builds the WebAssembly binaries +./scripts/build.sh # Builds the WebAssembly binaries cargo build # Builds all native code ---- @@ -118,7 +118,7 @@ The easiest/faster option is to use the latest image. Let´s first check the version we have. The first time you run this command, the polkadot docker image will be downloaded. This takes a bit of time and bandwidth, be patient: [source, shell] -docker run --rm -it chevdor/polkadot:latest ./version +docker run --rm -it chevdor/polkadot:latest pokadot --version .Polkadot arguments @@ -147,10 +147,11 @@ To update: To get up and running with the smallest footprint on your system, you may use the Polkadot Docker image. You can either build it yourself (it takes a while...): +NOTE: Make sure you build from the project root, *not* from the `./docker` folder. + [source, shell] ---- -ccd docker -./build.sh +./docker/build.sh ---- === Reporting issues diff --git a/ci/script.sh b/ci/script.sh index 8d8ded7d7bdd8..dafe2ad5a6d9c 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -22,7 +22,8 @@ case $TARGET in "wasm") # Install prerequisites and build all wasm projects - ./init.sh - ./build.sh + ./scripts/init.sh + ./scripts/build.sh + ./scripts/build-demos.sh ;; esac diff --git a/docker/Dockerfile b/docker/Dockerfile index dfc4ff77877b1..d5ab41727dd95 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,33 +1,28 @@ -FROM phusion/baseimage:0.10.1 +FROM rust:latest as builder LABEL maintainer "chevdor@gmail.com" +LABEL description="This is the build stage for Polkadot. Here we create the binary." ARG PROFILE=release +WORKDIR /polkadot +COPY . . +RUN cargo build --$PROFILE + -RUN mkdir -p polkadot && \ - apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y cmake pkg-config libssl-dev git && \ - apt-get clean && \ +FROM ubuntu:latest +LABEL maintainer "chevdor@gmail.com" +LABEL description="This is the 2nd stage: a very small image where we copy the Polkadot binary." +ARG PROFILE=release +COPY --from=builder /polkadot/target/$PROFILE/polkadot /usr/local/bin + +RUN apt-get update && \ + apt-get install -y \ + libssl1.1 && \ + apt-get clean && \ + apt-get autoclean && \ mkdir -p /root/.local/share/Polkadot && \ ln -s /root/.local/share/Polkadot /data -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ - export PATH=$PATH:$HOME/.cargo/bin && \ - rustup update nightly && \ - rustup target add wasm32-unknown-unknown --toolchain nightly && \ - rustup update stable && \ - cargo install --git https://github.com/alexcrichton/wasm-gc && \ - git clone https://github.com/paritytech/polkadot.git && \ - cd polkadot && \ - ./build.sh && \ - cargo build --$PROFILE && \ - mv target/$PROFILE/polkadot /usr/local/bin && \ - cargo clean && \ - rm -rf /root/.cargo /root/.rustup /tmp/* - -COPY version /polkadot -WORKDIR /polkadot EXPOSE 30333 9933 9944 VOLUME ["/data"] -CMD ["/bin/sh", "polkadot"] +CMD ["/usr/local/bin/polkadot"] diff --git a/docker/build.sh b/docker/build.sh index fdbe8c3f8108c..a4c6831676ff2 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -1,18 +1,26 @@ #!/usr/bin/env bash set -e +pushd . + +# The following line ensure we run from the project root +PROJECT_ROOT=`git rev-parse --show-toplevel` +cd $PROJECT_ROOT + # Find the current version from Cargo.toml -VERSION=`grep "^version" ../Cargo.toml | egrep -o "([0-9\.]+)"` +VERSION=`grep "^version" ./Cargo.toml | egrep -o "([0-9\.]+)"` GITUSER=chevdor GITREPO=polkadot # Build the image -echo "Building ${GITREPO}:$VERSION docker image, hang on!" -time docker build --build-arg PROFILE=release -t ${GITUSER}/${GITREPO}:$VERSION . +echo "Building ${GITUSER}/${GITREPO}:latest docker image, hang on!" +time docker build -f ./docker/Dockerfile --build-arg PROFILE=release -t ${GITUSER}/${GITREPO}:latest . # Show the list of available images for this repo echo "Image is ready" docker images | grep ${GITREPO} -echo -e "\nIf you just built the latest, you may want to update your tag:" -echo " $ docker tag ${GITUSER}/${GITREPO}:$VERSION ${GITUSER}/${GITREPO}:latest" +echo -e "\nIf you just built version ${VERSION}, you may want to update your tag:" +echo " $ docker tag ${GITUSER}/${GITREPO}:$VERSION ${GITUSER}/${GITREPO}:${VERSION}" + +popd diff --git a/docker/cleanup.sh b/docker/cleanup.sh deleted file mode 100755 index b4de473a1b54f..0000000000000 --- a/docker/cleanup.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# This script helps reduce the size of the built image -# It removes data that is not required. - -export PATH=$PATH:$HOME/.cargo/bin - -cargo clean -rm -rf /root/.cargo /root/.rustup /tmp/* diff --git a/docker/readme-docker.adoc b/docker/readme-docker.adoc index bbaceacd7f5c6..f6f105f1dd812 100644 --- a/docker/readme-docker.adoc +++ b/docker/readme-docker.adoc @@ -11,6 +11,6 @@ Run the following command To build your own image from the source, you can run the following command: - ./build.sh + ./docker/build.sh NOTE: Building the image takes a while. Count at least 30min on a good machine. diff --git a/docker/version b/docker/version deleted file mode 100755 index 047da3302ab95..0000000000000 --- a/docker/version +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# This script show the polkadot version and commit ref that was -# used to build the image. -# If you report an issue, call this script to get all details. -# This script will no longer be required once the polkadot cli -# can report its commit ref. - -echo "-----------------------------------------" -printf "Polkadot Docker Container: " -polkadot --version -printf " " -git rev-parse HEAD -echo "-----------------------------------------" diff --git a/scripts/build-demos.sh b/scripts/build-demos.sh new file mode 100755 index 0000000000000..5d5738cf78acb --- /dev/null +++ b/scripts/build-demos.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# This script assumes that all pre-requisites are installed. + +set -e + +source `dirname "$0"`/common.sh + +export CARGO_INCREMENTAL=0 + +# Save current directory. +pushd . + +cd $ROOT + +for DEMO in "${DEMOS[@]}" +do + echo "*** Building wasm binaries in $DEMO" + cd $DEMO + + ./build.sh + + cd - >> /dev/null +done + +# Restore initial directory. +popd diff --git a/build.sh b/scripts/build.sh similarity index 100% rename from build.sh rename to scripts/build.sh diff --git a/common.sh b/scripts/common.sh similarity index 97% rename from common.sh rename to scripts/common.sh index 847aa23820439..edaae85fc0d25 100644 --- a/common.sh +++ b/scripts/common.sh @@ -6,6 +6,9 @@ ROOT=`dirname "$0"` SRCS=( "polkadot/runtime/wasm" "substrate/executor/wasm" +) + +DEMOS=( "demo/runtime/wasm" "substrate/test-runtime/wasm" "polkadot/parachain/test-chains/basic_add" diff --git a/init.sh b/scripts/init.sh similarity index 86% rename from init.sh rename to scripts/init.sh index 2bd46709b8e88..e3618783c14a7 100755 --- a/init.sh +++ b/scripts/init.sh @@ -2,7 +2,7 @@ set -e -echo "*** Initilising WASM build environment" +echo "*** Initialising WASM build environment" rustup update nightly rustup target add wasm32-unknown-unknown --toolchain nightly From 08414edb88a50d5fca85935e11c00a93326f07b9 Mon Sep 17 00:00:00 2001 From: chevdor Date: Tue, 31 Jul 2018 19:02:21 +0200 Subject: [PATCH 2/4] Minor doc updates --- README.adoc | 4 +--- docker/readme-docker.adoc | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.adoc b/README.adoc index 943b0d0c68ac6..07a864c55a075 100644 --- a/README.adoc +++ b/README.adoc @@ -147,8 +147,6 @@ To update: To get up and running with the smallest footprint on your system, you may use the Polkadot Docker image. You can either build it yourself (it takes a while...): -NOTE: Make sure you build from the project root, *not* from the `./docker` folder. - [source, shell] ---- ./docker/build.sh @@ -160,7 +158,7 @@ If you run into issues with polkadot when using docker, please run the following (replace the tag with the appropriate one if you do not use latest): [source, shell] -docker run --rm -it chevdor/polkadot:latest version +docker run --rm -it chevdor/polkadot:latest polkadot --version This will show you the polkadot version as well as the git commit ref that was used to build your container. Just paste that in the issue you create. diff --git a/docker/readme-docker.adoc b/docker/readme-docker.adoc index f6f105f1dd812..6e41f792cc22b 100644 --- a/docker/readme-docker.adoc +++ b/docker/readme-docker.adoc @@ -5,7 +5,7 @@ Run the following command - docker run -d chevdor/polkadot:latest polkadot + docker run -d -P --name polkadot chevdor/polkadot:latest === Building the image From c73cee3d8854758af169df892735c180b75f4f60 Mon Sep 17 00:00:00 2001 From: chevdor Date: Thu, 2 Aug 2018 14:56:36 +0200 Subject: [PATCH 3/4] Fix and reduce size of the docker image --- Dockerfile | 42 ++++++++++++++++++++++++++++++++++++++++++ docker/Dockerfile | 28 ---------------------------- 2 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 Dockerfile delete mode 100644 docker/Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000..629b76454428f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +FROM phusion/baseimage:0.10.1 as builder +LABEL maintainer "chevdor@gmail.com" +LABEL description="This is the build stage for Polkadot. Here we create the binary." + +ARG PROFILE=release +WORKDIR /polkadot + +COPY . /polkadot + +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y cmake pkg-config libssl-dev git + +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ + export PATH=$PATH:$HOME/.cargo/bin && \ + # rustup update nightly && \ + # rustup target add wasm32-unknown-unknown --toolchain nightly && \ + # rustup update stable && \ + # cargo install --git https://github.com/alexcrichton/wasm-gc && \ + cargo build --$PROFILE + +# ===== SECOND STAGE ====== + +FROM phusion/baseimage:0.10.0 +LABEL maintainer "chevdor@gmail.com" +LABEL description="This is the 2nd stage: a very small image where we copy the Polkadot binary." +ARG PROFILE=release +COPY --from=builder /polkadot/target/$PROFILE/polkadot /usr/local/bin + +RUN mv /usr/share/ca* /tmp && \ + rm -rf /usr/share/* && \ + mv /tmp/ca-certificates /usr/share/ && \ + rm -rf /usr/lib/python* && \ + mkdir -p /root/.local/share/Polkadot && \ + ln -s /root/.local/share/Polkadot /data + +RUN rm -rf /usr/bin /usr/sbin + +EXPOSE 30333 9933 9944 +VOLUME ["/data"] + +CMD ["/usr/local/bin/polkadot"] diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index d5ab41727dd95..0000000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM rust:latest as builder -LABEL maintainer "chevdor@gmail.com" -LABEL description="This is the build stage for Polkadot. Here we create the binary." - -ARG PROFILE=release -WORKDIR /polkadot -COPY . . -RUN cargo build --$PROFILE - - -FROM ubuntu:latest -LABEL maintainer "chevdor@gmail.com" -LABEL description="This is the 2nd stage: a very small image where we copy the Polkadot binary." -ARG PROFILE=release -COPY --from=builder /polkadot/target/$PROFILE/polkadot /usr/local/bin - -RUN apt-get update && \ - apt-get install -y \ - libssl1.1 && \ - apt-get clean && \ - apt-get autoclean && \ - mkdir -p /root/.local/share/Polkadot && \ - ln -s /root/.local/share/Polkadot /data - -EXPOSE 30333 9933 9944 -VOLUME ["/data"] - -CMD ["/usr/local/bin/polkadot"] From 5721984fa19b8682010c3ca1d358254b35e629e9 Mon Sep 17 00:00:00 2001 From: chevdor Date: Thu, 2 Aug 2018 15:01:29 +0200 Subject: [PATCH 4/4] Fix paths in scripts --- Dockerfile | 4 ---- scripts/build-demos.sh | 3 ++- scripts/build.sh | 3 ++- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 629b76454428f..e07e647ce41c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,10 +13,6 @@ RUN apt-get update && \ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ export PATH=$PATH:$HOME/.cargo/bin && \ - # rustup update nightly && \ - # rustup target add wasm32-unknown-unknown --toolchain nightly && \ - # rustup update stable && \ - # cargo install --git https://github.com/alexcrichton/wasm-gc && \ cargo build --$PROFILE # ===== SECOND STAGE ====== diff --git a/scripts/build-demos.sh b/scripts/build-demos.sh index 5d5738cf78acb..285da143c17d8 100755 --- a/scripts/build-demos.sh +++ b/scripts/build-demos.sh @@ -4,6 +4,7 @@ set -e +PROJECT_ROOT=`git rev-parse --show-toplevel` source `dirname "$0"`/common.sh export CARGO_INCREMENTAL=0 @@ -16,7 +17,7 @@ cd $ROOT for DEMO in "${DEMOS[@]}" do echo "*** Building wasm binaries in $DEMO" - cd $DEMO + cd "$PROJECT_ROOT/$DEMO" ./build.sh diff --git a/scripts/build.sh b/scripts/build.sh index f2ced1300ac03..9cacf74dbb07a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,6 +4,7 @@ set -e +PROJECT_ROOT=`git rev-parse --show-toplevel` source `dirname "$0"`/common.sh export CARGO_INCREMENTAL=0 @@ -16,7 +17,7 @@ cd $ROOT for SRC in "${SRCS[@]}" do echo "*** Building wasm binaries in $SRC" - cd $SRC + cd "$PROJECT_ROOT/$SRC" ./build.sh