From 3ce39559b24ac077e8c463212743598b56240f13 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 13 May 2022 22:07:45 +0200 Subject: [PATCH 1/7] update Ruby to 2.6.10, fix Gemfile and push Gemfile.lock Signed-off-by: CrazyMax --- .dockerignore | 7 +---- .gitignore | 2 +- .ruby-version | 2 +- Gemfile | 51 ++++++--------------------------- Gemfile.lock | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 51 deletions(-) create mode 100644 Gemfile.lock diff --git a/.dockerignore b/.dockerignore index 256a36839e3d..0ca8aaa66b16 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,7 @@ -.dockerignore .DS_Store .git .github .gitignore -.gitmodules .idea .jekyll-metadata .sass-cache @@ -11,9 +9,6 @@ tests _site CONTRIBUTING.md Dockerfile -Dockerfile.archive docker-compose.yml -Gemfile -Gemfile.lock _website*.json - +/vendor diff --git a/.gitignore b/.gitignore index 010dcf665753..52ae55e39429 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,5 @@ _site/** .sass-cache/** CNAME -Gemfile.lock _kbase/** +/vendor diff --git a/.ruby-version b/.ruby-version index e75da3e63d60..a04abec91494 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.3.6 +2.6.10 diff --git a/Gemfile b/Gemfile index f5ffef68d348..18df52ad127c 100644 --- a/Gemfile +++ b/Gemfile @@ -1,43 +1,8 @@ -source "https://rubygems.org" - -# Update me once in a while: https://github.com/github/pages-gem/releases -# Please ensure, before upgrading, that this version exists as a tag in starefossen/github-pages here: -# https://hub.docker.com/r/starefossen/github-pages/tags/ -# -# Fresh install? -# -# Windows: -# Install Ruby 2.3.3 x64 and download the Development Kit for 64-bit: -# https://rubyinstaller.org/downloads/ -# -# Run this to install devkit after extracting: -# ruby /dk.rb init -# ruby /dk.rb install -# -# then: -# gem install bundler -# bundle install -# -# Mac/Linux: -# Install Ruby 2.3.x and then: -# gem install bundler -# bundle install -# -# --------------------- -# Upgrading? Probably best to reset your environment: -# -# Remove all gems: -# gem uninstall -aIx -# -# (If Windows, do the dk.rb bits above, then go to the next step below) - -# Install anew: -# gem install bundler -# bundle install - -# This only affects interactive builds (local build, Netlify) and not the -# live site deploy, which uses the Dockerfiles found in the publish-tools -# branch. - -gem "github-pages", "198" -gem 'wdm' if Gem.win_platform? +source 'https://rubygems.org' + +gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] +gem 'wdm', '>= 0.1.0' if Gem.win_platform? +gem 'jekyll' +gem 'jekyll-redirect-from' +gem 'jekyll-relative-links' +gem 'jekyll-sitemap' diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 000000000000..588ff7a64d92 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,79 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.0) + public_suffix (>= 2.0.2, < 5.0) + colorator (1.1.0) + concurrent-ruby (1.1.10) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + eventmachine (1.2.7) + ffi (1.15.5) + forwardable-extended (2.6.0) + http_parser.rb (0.8.0) + i18n (1.10.0) + concurrent-ruby (~> 1.0) + jekyll (4.2.2) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (~> 1.0) + jekyll-sass-converter (~> 2.0) + jekyll-watch (~> 2.0) + kramdown (~> 2.3) + kramdown-parser-gfm (~> 1.0) + liquid (~> 4.0) + mercenary (~> 0.4.0) + pathutil (~> 0.9) + rouge (~> 3.0) + safe_yaml (~> 1.0) + terminal-table (~> 2.0) + jekyll-redirect-from (0.16.0) + jekyll (>= 3.3, < 5.0) + jekyll-relative-links (0.6.1) + jekyll (>= 3.3, < 5.0) + jekyll-sass-converter (2.2.0) + sassc (> 2.0.1, < 3.0) + jekyll-sitemap (1.4.0) + jekyll (>= 3.7, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + kramdown (2.4.0) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.3) + listen (3.7.1) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.4.0) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (4.0.7) + rb-fsevent (0.11.1) + rb-inotify (0.10.1) + ffi (~> 1.0) + rexml (3.2.5) + rouge (3.28.0) + safe_yaml (1.0.5) + sassc (2.4.0) + ffi (~> 1.9) + terminal-table (2.0.0) + unicode-display_width (~> 1.1, >= 1.1.1) + unicode-display_width (1.8.0) + +PLATFORMS + aarch64-linux + arm-linux + x86_64-linux + +DEPENDENCIES + jekyll + jekyll-redirect-from + jekyll-relative-links + jekyll-sitemap + tzinfo-data + +BUNDLED WITH + 2.3.13 From 1e1fdfce1c291769686fbd1884e0fb883ef44c3d Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 6 May 2022 17:12:59 +0200 Subject: [PATCH 2/7] ignore .jekyll-cache folder Signed-off-by: CrazyMax --- .dockerignore | 1 + .gitignore | 1 + 2 files changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 0ca8aaa66b16..6b5119886961 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ .github .gitignore .idea +.jekyll-cache .jekyll-metadata .sass-cache tests diff --git a/.gitignore b/.gitignore index 52ae55e39429..66fb99c3ff76 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ **/.DS_Store **/desktop.ini .bundle/** +.jekyll-cache .jekyll-metadata _site/** .sass-cache/** From ebd36f70818d3ecfbcd9943f75ab7f6fd7f9dac2 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 13 May 2022 21:24:26 +0200 Subject: [PATCH 3/7] dockerfile: enhanced to match netlify pipeline Signed-off-by: CrazyMax --- Dockerfile | 163 +++++++++++++++++++++++++----------------------- Makefile | 16 +++++ docker-bake.hcl | 21 +++++++ 3 files changed, 122 insertions(+), 78 deletions(-) create mode 100644 docker-bake.hcl diff --git a/Dockerfile b/Dockerfile index 1505b53c8399..3c8a602390cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,58 +1,55 @@ +# syntax=docker/dockerfile:1 + # This Dockerfile builds the docs for https://docs.docker.com/ # from the master branch of https://github.com/docker/docker.github.io -# -# Here is the sequence: -# 1. Set up base stages for building and deploying -# 2. Collect and build the reference documentation (from upstream resources) -# 3. Build static HTML from the current branch -# 4. Build the final image, combining the reference docs and current version -# of the documentation -# -# When the image is run, it starts Nginx and serves the docs at port 4000 - -# Jekyll environment (development/production) + +# Use same ruby version as the one in .ruby-version +# that is used by Netlify +ARG RUBY_VERSION=2.6.10 +# Same as the one in Gemfile.lock +ARG BUNDLER_VERSION=2.3.13 + ARG JEKYLL_ENV=development +ARG DOMAIN=docs.docker.com -# Engine ARG ENGINE_BRANCH="20.10" - -# Distribution ARG DISTRIBUTION_BRANCH="release/2.7" - -# Compose CLI ARG COMPOSE_CLI_BRANCH="main" - -# extensions SDK ARG EXTENSIONS_SDK_BRANCH="main" -### -# Set up base stages for building and deploying -### -FROM starefossen/github-pages:198 AS builderbase -ENV TARGET=/usr/share/nginx/html -WORKDIR /usr/src/app/md_source/ - -# Set vars used by fetch-upstream-resources.sh script as an environment variable, -# so that they are persisted in the image for use in later stages. -ARG ENGINE_BRANCH -ENV ENGINE_BRANCH=${ENGINE_BRANCH} - -ARG DISTRIBUTION_BRANCH -ENV DISTRIBUTION_BRANCH=${DISTRIBUTION_BRANCH} - -ARG COMPOSE_CLI_BRANCH -ENV COMPOSE_CLI_BRANCH=${COMPOSE_CLI_BRANCH} - -ARG EXTENSIONS_SDK_BRANCH -ENV EXTENSIONS_SDK_BRANCH=${EXTENSIONS_SDK_BRANCH} +# Base stage for building +FROM ruby:${RUBY_VERSION}-alpine AS base +WORKDIR /src +RUN apk add --no-cache bash build-base git subversion wget + +# Gem stage will install bundler used as dependency manager +# for our dependencies in Gemfile for Jekyll +FROM base AS gem +ARG BUNDLER_VERSION +COPY Gemfile* . +RUN gem uninstall -aIx bundler \ + && gem install bundler -v ${BUNDLER_VERSION} \ + && bundle install --jobs 4 --retry 3 + +# Vendor Gemfile for Jekyll +FROM gem AS vendored +ARG BUNDLER_VERSION +RUN bundle update \ + && mkdir /out \ + && cp Gemfile.lock /out + +# Stage used to output the vendored Gemfile.lock: +# > make vendor +# or +# > docker buildx bake vendor +FROM scratch AS vendor +COPY --from=vendored /out / # Fetch upstream resources (reference documentation) # Only add the files that are needed to build these reference docs, so that these # docs are only rebuilt if changes were made to ENGINE_BRANCH or DISTRIBUTION_BRANCH. -# Disable caching (docker build --no-cache) to force updating these docs. -FROM alpine AS upstream-resources -RUN apk add --no-cache subversion wget -WORKDIR /usr/src/app/md_source/ +FROM base AS upstream-resources +WORKDIR /out COPY ./_scripts/fetch-upstream-resources.sh ./_scripts/ ARG ENGINE_BRANCH ARG DISTRIBUTION_BRANCH @@ -60,49 +57,59 @@ ARG COMPOSE_CLI_BRANCH ARG EXTENSIONS_SDK_BRANCH RUN ./_scripts/fetch-upstream-resources.sh . - # Build the static HTML for the current docs. # After building with jekyll, fix up some links -FROM builderbase AS current +FROM gem AS generate +ARG JEKYLL_ENV +ARG DOMAIN +ENV TARGET=/out COPY . . -COPY --from=upstream-resources /usr/src/app/md_source/. ./ +COPY --from=upstream-resources /out . +RUN --mount=type=cache,target=/src/.jekyll-cache </#https://docs.docker.com/#' "${TARGET}/sitemap.xml"; \ - else \ - jekyll build --profile -d ${TARGET}; \ - echo '[]' > ${TARGET}/js/metadata.json; \ - fi; \ - find ${TARGET} -type f -name '*.html' | while read i; do sed -i 's#\(]* href="\)https://docs.docker.com/#\1/#g' "$i"; done; - - -# This stage only contains the generated files. It can be used to host the -# documentation on a non-containerised service (e.g. to deploy to an s3 bucket). -# When using BuildKit, use the '--output' option to build the files and to copy -# them to your local filesystem. -# -# DOCKER_BUILDKIT=1 docker build --target=deploy-source --output=./_site . -FROM scratch AS deploy-source -COPY --from=current /usr/share/nginx/html / - -# Final stage, which includes nginx, and the current docs. -# -# To build current docs: -# DOCKER_BUILDKIT=1 docker build -t docs . +(set -x ; ./_scripts/update-api-toc.sh) + +if [ "${JEKYLL_ENV}" = "production" ]; then + ( + set -x + bundle exec jekyll build --profile -d ${TARGET} --config _config.yml,_config_production.yml + sed -i 's#/#https://${DOMAIN}/#' "${TARGET}/sitemap.xml" + ) +else + ( + set -x + bundle exec jekyll build --trace --profile -d ${TARGET} + mkdir -p ${TARGET}/js + echo '[]' > ${TARGET}/js/metadata.json + ) +fi + +find ${TARGET} -type f -name '*.html' | while read i; do + sed -i 's#\(]* href="\)https://${DOMAIN}/#\1/#g' "$i" +done +EOT + +# Release the generated files in a scratch image +# Can be output to your host with: +# > make release +# or +# > docker buildx bake release +FROM scratch AS release +COPY --from=generate /out / + +# Create a runnable nginx instance with generated HTML files. +# When the image is run, it starts Nginx and serves the docs at port 4000: +# > make deploy +# or +# > docker-compose up --build FROM nginx:alpine AS deploy -ENV TARGET=/usr/share/nginx/html -WORKDIR $TARGET - -COPY --from=current /usr/share/nginx/html . - -# Configure NGINX +COPY --from=release / /usr/share/nginx/html COPY _deploy/nginx/default.conf /etc/nginx/conf.d/default.conf ARG JEKYLL_ENV ENV JEKYLL_ENV=${JEKYLL_ENV} CMD echo -e "Docker docs are viewable at:\nhttp://0.0.0.0:4000 (build target: ${JEKYLL_ENV})"; exec nginx -g 'daemon off;' + +FROM deploy diff --git a/Makefile b/Makefile index 9b3c6f1275e1..f1e20b602284 100644 --- a/Makefile +++ b/Makefile @@ -19,3 +19,19 @@ buildx-yaml: rm -rf ./_data/buildx/* cp -R "$($@_TMP_OUT)"/out/reference/*.yaml ./_data/buildx/ rm -rf $($@_TMP_OUT)/* + +# Build website and output to _site folder +release: + rm -rf _site + $(BUILDX_CMD) bake release + +# Vendor Gemfile.lock +vendor: + $(BUILDX_CMD) bake vendor + +# Deploy website and run it through Docker compose +# Available in your browser at http://localhost:4000 +deploy: + docker compose up --build + +.PHONY: buildx-yaml release vendor deploy diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 000000000000..ac5782529038 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,21 @@ +variable "JEKYLL_ENV" { + default = "development" +} + +group "default" { + targets = ["release"] +} + +target "release" { + target = "release" + args = { + JEKYLL_ENV = JEKYLL_ENV + } + no-cache-filter = ["upstream-resources"] + output = ["./_site"] +} + +target "vendor" { + target = "vendor" + output = ["."] +} From 75cb114327fc7115ba85c85c847d23ee53850020 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 6 May 2022 17:50:15 +0200 Subject: [PATCH 4/7] ci: update github actions accordingly to the new dockerfile Signed-off-by: CrazyMax --- .github/workflows/build-master.yml | 44 ++++++++++++------------ .github/workflows/build-pr.yml | 42 ++++++++++++----------- .github/workflows/build-published.yml | 48 +++++++++++++++------------ 3 files changed, 71 insertions(+), 63 deletions(-) diff --git a/.github/workflows/build-master.yml b/.github/workflows/build-master.yml index 6a8252179c21..fa8d9079ed98 100644 --- a/.github/workflows/build-master.yml +++ b/.github/workflows/build-master.yml @@ -6,25 +6,27 @@ on: - master jobs: - build-static-page: - name: build - runs-on: ubuntu-18.04 - timeout-minutes: 10 - env: - DOCKER_BUILDKIT: '1' + build: + runs-on: ubuntu-20.04 + if: github.repository == 'docker/docker.github.io' steps: - - name: print docker info - run: docker version && docker info - - uses: actions/checkout@v2 - - name: build current docs - run: docker build --target=deploy-source --output=./_site . - - name: upload files to S3 bucket - run: aws s3 sync --acl public-read _site s3://docs.docker.com-stage-us-east-1/ --delete + - + name: Checkout + uses: actions/checkout@v2 + - + name: Build + uses: docker/bake-action@v2 + with: + targets: release + - + name: Upload files to S3 bucket + run: | + aws s3 sync --acl public-read _site s3://docs.docker.com-stage-us-east-1/ --delete env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.repository == 'docker/docker.github.io' && github.event_name == 'push' - - name: update S3 website config + - + name: Update S3 website config uses: ./.github/actions/update-website-config with: bucketName: docs.docker.com-stage-us-east-1 @@ -33,16 +35,16 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.repository == 'docker/docker.github.io' && github.event_name == 'push' - - name: invalidate docs website cache - run: aws --region us-east-1 lambda invoke --function-name arn:aws:lambda:us-east-1:710015040892:function:docs-stage-cache-invalidator response.json + - + name: Invalidate docs website cache + run: | + aws --region us-east-1 lambda invoke --function-name arn:aws:lambda:us-east-1:710015040892:function:docs-stage-cache-invalidator response.json env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.repository == 'docker/docker.github.io' && github.event_name == 'push' - - name: send slack notification + - + name: Send Slack notification run: | curl -X POST -H 'Content-type: application/json' --data '{"text":"Successfully promoted docs-stage from master. https://docs-stage.docker.com/"}' $SLACK env: SLACK: ${{ secrets.SLACK_WEBHOOK }} - if: github.repository == 'docker/docker.github.io' && github.event_name == 'push' diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index 56408735c64c..e5f5e84cd084 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -3,29 +3,31 @@ name: build docker image when PR is opened on: pull_request jobs: - build-static-page: - name: build - runs-on: ubuntu-18.04 - env: - DOCKER_BUILDKIT: '1' + build: + runs-on: ubuntu-20.04 steps: - - name: print docker info - run: docker version && docker info - - uses: actions/checkout@v2 - - name: build image - run: docker build --target=current -t documentation:latest . + - + name: Checkout + uses: actions/checkout@v2 + - + name: Build + uses: docker/bake-action@v2 + with: + targets: release + validate: - name: validate links - runs-on: ubuntu-18.04 - env: - DOCKER_BUILDKIT: '1' + runs-on: ubuntu-20.04 steps: - - name: print docker info - run: docker version && docker info - - uses: actions/checkout@v2 - - name: copy files to host - run: docker build --target=deploy-source --output=./_site . - - name: check for broken links + - + name: Checkout + uses: actions/checkout@v2 + - + name: Build + uses: docker/bake-action@v2 + with: + targets: release + - + name: Check for broken links uses: chabad360/htmlproofer@master with: directory: ./_site diff --git a/.github/workflows/build-published.yml b/.github/workflows/build-published.yml index 7c330c582c85..b5fd6a84e4db 100644 --- a/.github/workflows/build-published.yml +++ b/.github/workflows/build-published.yml @@ -6,25 +6,29 @@ on: - published jobs: - build-static-page: - name: build - runs-on: ubuntu-18.04 - timeout-minutes: 10 - env: - DOCKER_BUILDKIT: '1' + build: + runs-on: ubuntu-20.04 + if: github.repository == 'docker/docker.github.io' steps: - - name: print docker info - run: docker version && docker info - - uses: actions/checkout@v2 - - name: build current docs - run: docker build --build-arg JEKYLL_ENV=production --target=deploy-source --output=./_site . - - name: upload files to S3 bucket - run: aws s3 sync --acl public-read _site s3://docs.docker.com-us-east-1/ --delete + - + name: Checkout + uses: actions/checkout@v2 + - + name: Build + uses: docker/bake-action@v2 + with: + targets: release + env: + JEKYLL_ENV: production + - + name: Upload files to S3 bucket + run: | + aws s3 sync --acl public-read _site s3://docs.docker.com-us-east-1/ --delete env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.repository == 'docker/docker.github.io' && github.event_name == 'push' - - name: update S3 website config + - + name: Update S3 website config uses: ./.github/actions/update-website-config with: bucketName: docs.docker.com-us-east-1 @@ -32,17 +36,17 @@ jobs: websiteConfig: _website-config-docs.json env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.repository == 'docker/docker.github.io' && github.event_name == 'push' - - name: invalidate docs website cache - run: aws --region us-east-1 lambda invoke --function-name arn:aws:lambda:us-east-1:710015040892:function:docs-cache-invalidator response.json + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - + name: Invalidate docs website cache + run: | + aws --region us-east-1 lambda invoke --function-name arn:aws:lambda:us-east-1:710015040892:function:docs-cache-invalidator response.json env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: github.repository == 'docker/docker.github.io' && github.event_name == 'push' - - name: send slack notification + - + name: Send Slack notification run: | curl -X POST -H 'Content-type: application/json' --data '{"text":"Successfully published docs. https://docs.docker.com/"}' $SLACK env: SLACK: ${{ secrets.SLACK_WEBHOOK }} - if: github.repository == 'docker/docker.github.io' && github.event_name == 'push' From dc1a99b7b02e3d8a4b59bcb00ac932854804f5b3 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 13 May 2022 14:00:19 +0200 Subject: [PATCH 5/7] ci: merge workflows Signed-off-by: CrazyMax --- .github/workflows/build-master.yml | 50 ------------ .github/workflows/build-pr.yml | 49 ------------ .github/workflows/build-published.yml | 52 ------------ .github/workflows/build.yml | 111 ++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 151 deletions(-) delete mode 100644 .github/workflows/build-master.yml delete mode 100644 .github/workflows/build-pr.yml delete mode 100644 .github/workflows/build-published.yml create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build-master.yml b/.github/workflows/build-master.yml deleted file mode 100644 index fa8d9079ed98..000000000000 --- a/.github/workflows/build-master.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: deploy stage website from master branch - -on: - push: - branches: - - master - -jobs: - build: - runs-on: ubuntu-20.04 - if: github.repository == 'docker/docker.github.io' - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Build - uses: docker/bake-action@v2 - with: - targets: release - - - name: Upload files to S3 bucket - run: | - aws s3 sync --acl public-read _site s3://docs.docker.com-stage-us-east-1/ --delete - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Update S3 website config - uses: ./.github/actions/update-website-config - with: - bucketName: docs.docker.com-stage-us-east-1 - regionName: us-east-1 - websiteConfig: _website-config-docs-stage.json - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Invalidate docs website cache - run: | - aws --region us-east-1 lambda invoke --function-name arn:aws:lambda:us-east-1:710015040892:function:docs-stage-cache-invalidator response.json - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Send Slack notification - run: | - curl -X POST -H 'Content-type: application/json' --data '{"text":"Successfully promoted docs-stage from master. https://docs-stage.docker.com/"}' $SLACK - env: - SLACK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml deleted file mode 100644 index e5f5e84cd084..000000000000 --- a/.github/workflows/build-pr.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: build docker image when PR is opened - -on: pull_request - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Build - uses: docker/bake-action@v2 - with: - targets: release - - validate: - runs-on: ubuntu-20.04 - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Build - uses: docker/bake-action@v2 - with: - targets: release - - - name: Check for broken links - uses: chabad360/htmlproofer@master - with: - directory: ./_site - # for available options, refer to: - # - https://github.com/gjtorikian/html-proofer - # - https://github.com/gjtorikian/html-proofer/blob/main/bin/htmlproofer - arguments: --disable-external --internal-domains="docs.docker.com,docs-stage.docker.com,localhost:4000" --file-ignore="/^./_site/engine/api/.*$/,./_site/registry/configuration/index.html" --url-ignore="/^/docker-hub/api/latest/.*$/,/^/engine/api/v.+/#.*$/,/^/glossary/.*$/" - - # Disabled netlify-deploy due to flakey 502 http errors - # - name: copy static files - # if: github.event.pull_request.head.repo.fork == false - # run: docker run -v ${PWD}:/output documentation:latest cp -r /usr/share/nginx/html /output/_site - # - uses: ./.github/actions/netlify-deploy - # if: github.event.pull_request.head.repo.fork == false - # with: - # directory: _site - # netlify_token: ${{ secrets.NETLIFY_AUTH_TOKEN }} - # netlify_account_slug: ${{ secrets.NETLIFY_ACCOUNT_SLUG }} - # site_name: "${{ github.repository }}/${{ github.head_ref }}" diff --git a/.github/workflows/build-published.yml b/.github/workflows/build-published.yml deleted file mode 100644 index b5fd6a84e4db..000000000000 --- a/.github/workflows/build-published.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: deploy website from published branch - -on: - push: - branches: - - published - -jobs: - build: - runs-on: ubuntu-20.04 - if: github.repository == 'docker/docker.github.io' - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Build - uses: docker/bake-action@v2 - with: - targets: release - env: - JEKYLL_ENV: production - - - name: Upload files to S3 bucket - run: | - aws s3 sync --acl public-read _site s3://docs.docker.com-us-east-1/ --delete - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Update S3 website config - uses: ./.github/actions/update-website-config - with: - bucketName: docs.docker.com-us-east-1 - regionName: us-east-1 - websiteConfig: _website-config-docs.json - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Invalidate docs website cache - run: | - aws --region us-east-1 lambda invoke --function-name arn:aws:lambda:us-east-1:710015040892:function:docs-cache-invalidator response.json - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Send Slack notification - run: | - curl -X POST -H 'Content-type: application/json' --data '{"text":"Successfully published docs. https://docs.docker.com/"}' $SLACK - env: - SLACK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000000..eeb41824105c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,111 @@ +name: build + +on: + push: + branches: + - master + - published + pull_request: + +jobs: + release: + runs-on: ubuntu-20.04 + steps: + - + name: Prepare + run: | + JEKYLL_ENV=development + if [ "${{ github.ref }}" = "refs/heads/master" ]; then + DOCS_S3_HOST="docs.docker.com-stage-us-east-1" + DOCS_AWS_LAMBDA="arn:aws:lambda:us-east-1:710015040892:function:docs-stage-cache-invalidator" + DOCS_SLACK_MSG="Successfully promoted docs-stage from master. https://docs-stage.docker.com/" + DOCS_WEBCONFIG="_website-config-docs-stage.json" + elif [ "${{ github.ref }}" = "refs/heads/published" ]; then + JEKYLL_ENV=production + DOCS_S3_HOST="docs.docker.com-us-east-1" + DOCS_AWS_LAMBDA="arn:aws:lambda:us-east-1:710015040892:function:docs-cache-invalidator" + DOCS_SLACK_MSG="Successfully published docs. https://docs.docker.com/" + DOCS_WEBCONFIG="_website-config-docs.json" + fi + echo "JEKYLL_ENV=$JEKYLL_ENV" >> $GITHUB_ENV + echo "DOCS_S3_HOST=$DOCS_S3_HOST" >> $GITHUB_ENV + echo "DOCS_AWS_LAMBDA=$DOCS_AWS_LAMBDA" >> $GITHUB_ENV + echo "DOCS_SLACK_MSG=$DOCS_SLACK_MSG" >> $GITHUB_ENV + echo "DOCS_WEBCONFIG=$DOCS_WEBCONFIG" >> $GITHUB_ENV + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Build + uses: docker/bake-action@v2 + with: + targets: release + - + name: Upload files to S3 bucket + if: github.event_name != 'pull_request' + run: | + aws s3 sync --acl public-read _site s3://${{ env.DOCS_S3_HOST }}/ --delete + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - + name: Update S3 website config + if: github.event_name != 'pull_request' + uses: ./.github/actions/update-website-config + with: + bucketName: ${{ env.DOCS_S3_HOST }} + regionName: us-east-1 + websiteConfig: ${{ env.DOCS_WEBCONFIG }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - + name: Invalidate docs website cache + if: github.event_name != 'pull_request' + run: | + aws --region us-east-1 lambda invoke --function-name ${{ env.DOCS_AWS_LAMBDA }} response.json + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - + name: Send Slack notification + if: github.event_name != 'pull_request' + run: | + curl -X POST -H 'Content-type: application/json' --data '{"text":"${{ env.DOCS_SLACK_MSG }}"}' ${{ secrets.SLACK_WEBHOOK }} + + validate: + runs-on: ubuntu-20.04 + if: github.event_name == 'pull_request' + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Build + uses: docker/bake-action@v2 + with: + targets: release + - + name: Check for broken links + uses: chabad360/htmlproofer@master + with: + directory: ./_site + # for available options, refer to: + # - https://github.com/gjtorikian/html-proofer + # - https://github.com/gjtorikian/html-proofer/blob/main/bin/htmlproofer + arguments: --disable-external --internal-domains="docs.docker.com,docs-stage.docker.com,localhost:4000" --file-ignore="/^./_site/engine/api/.*$/,./_site/registry/configuration/index.html" --url-ignore="/^/docker-hub/api/latest/.*$/,/^/engine/api/v.+/#.*$/,/^/glossary/.*$/" + + # Disabled netlify-deploy due to flakey 502 http errors + # - name: copy static files + # if: github.event.pull_request.head.repo.fork == false + # run: docker run -v ${PWD}:/output documentation:latest cp -r /usr/share/nginx/html /output/_site + # - uses: ./.github/actions/netlify-deploy + # if: github.event.pull_request.head.repo.fork == false + # with: + # directory: _site + # netlify_token: ${{ secrets.NETLIFY_AUTH_TOKEN }} + # netlify_account_slug: ${{ secrets.NETLIFY_ACCOUNT_SLUG }} + # site_name: "${{ github.repository }}/${{ github.head_ref }}" From 8da7d9445d1c1d0ba5108085ea8c93889bfa66b3 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Mon, 9 May 2022 15:08:40 +0200 Subject: [PATCH 6/7] dockerfile: containerized htmlproofer Signed-off-by: CrazyMax --- .github/workflows/build.yml | 13 ++----------- Dockerfile | 9 +++++++++ Gemfile | 1 + Gemfile.lock | 19 +++++++++++++++++++ Makefile | 6 +++++- docker-bake.hcl | 8 ++++++++ 6 files changed, 44 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eeb41824105c..10357654cf21 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,20 +83,11 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - - name: Build - uses: docker/bake-action@v2 - with: - targets: release - name: Check for broken links - uses: chabad360/htmlproofer@master + uses: docker/bake-action@v2 with: - directory: ./_site - # for available options, refer to: - # - https://github.com/gjtorikian/html-proofer - # - https://github.com/gjtorikian/html-proofer/blob/main/bin/htmlproofer - arguments: --disable-external --internal-domains="docs.docker.com,docs-stage.docker.com,localhost:4000" --file-ignore="/^./_site/engine/api/.*$/,./_site/registry/configuration/index.html" --url-ignore="/^/docker-hub/api/latest/.*$/,/^/engine/api/v.+/#.*$/,/^/glossary/.*$/" + targets: htmlproofer # Disabled netlify-deploy due to flakey 502 http errors # - name: copy static files diff --git a/Dockerfile b/Dockerfile index 3c8a602390cb..5f779e227f5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,6 +92,15 @@ find ${TARGET} -type f -name '*.html' | while read i; do done EOT +# htmlproofer checks for broken links +FROM gem AS htmlproofer +RUN --mount=type=bind,from=generate,source=/out,target=_site \ + htmlproofer ./_site \ + --disable-external \ + --internal-domains="docs.docker.com,docs-stage.docker.com,localhost:4000" \ + --file-ignore="/^./_site/engine/api/.*$/,./_site/registry/configuration/index.html" \ + --url-ignore="/^/docker-hub/api/latest/.*$/,/^/engine/api/v.+/#.*$/,/^/glossary/.*$/" + # Release the generated files in a scratch image # Can be output to your host with: # > make release diff --git a/Gemfile b/Gemfile index 18df52ad127c..231ad6d35935 100644 --- a/Gemfile +++ b/Gemfile @@ -6,3 +6,4 @@ gem 'jekyll' gem 'jekyll-redirect-from' gem 'jekyll-relative-links' gem 'jekyll-sitemap' +gem 'html-proofer' diff --git a/Gemfile.lock b/Gemfile.lock index 588ff7a64d92..af016bebda55 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,9 +8,19 @@ GEM em-websocket (0.5.3) eventmachine (>= 0.12.9) http_parser.rb (~> 0) + ethon (0.15.0) + ffi (>= 1.15.0) eventmachine (1.2.7) ffi (1.15.5) forwardable-extended (2.6.0) + html-proofer (3.19.3) + addressable (~> 2.3) + mercenary (~> 0.3) + nokogiri (~> 1.12) + parallel (~> 1.3) + rainbow (~> 3.0) + typhoeus (~> 1.3) + yell (~> 2.0) http_parser.rb (0.8.0) i18n (1.10.0) concurrent-ruby (~> 1.0) @@ -48,9 +58,14 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.4.0) + nokogiri (1.13.5-x86_64-linux) + racc (~> 1.4) + parallel (1.22.1) pathutil (0.16.2) forwardable-extended (~> 2.6) public_suffix (4.0.7) + racc (1.6.0) + rainbow (3.1.1) rb-fsevent (0.11.1) rb-inotify (0.10.1) ffi (~> 1.0) @@ -61,7 +76,10 @@ GEM ffi (~> 1.9) terminal-table (2.0.0) unicode-display_width (~> 1.1, >= 1.1.1) + typhoeus (1.4.0) + ethon (>= 0.9.0) unicode-display_width (1.8.0) + yell (2.2.2) PLATFORMS aarch64-linux @@ -69,6 +87,7 @@ PLATFORMS x86_64-linux DEPENDENCIES + html-proofer jekyll jekyll-redirect-from jekyll-relative-links diff --git a/Makefile b/Makefile index f1e20b602284..d93c87177cb3 100644 --- a/Makefile +++ b/Makefile @@ -29,9 +29,13 @@ release: vendor: $(BUILDX_CMD) bake vendor +# Check for broken links +htmlproofer: + $(BUILDX_CMD) bake htmlproofer + # Deploy website and run it through Docker compose # Available in your browser at http://localhost:4000 deploy: docker compose up --build -.PHONY: buildx-yaml release vendor deploy +.PHONY: buildx-yaml release vendor htmlproofer deploy diff --git a/docker-bake.hcl b/docker-bake.hcl index ac5782529038..aff41a86bf3f 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -19,3 +19,11 @@ target "vendor" { target = "vendor" output = ["."] } + +target "htmlproofer" { + target = "htmlproofer" + args = { + JEKYLL_ENV = JEKYLL_ENV + } + output = ["type=cacheonly"] +} From 0f493a16cdcc03ae1dd6fb95a84cc15bccf4473c Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 6 May 2022 21:17:34 +0200 Subject: [PATCH 7/7] jekyll: fix exclusions list Signed-off-by: CrazyMax --- _config.yml | 14 +++++++++++++- _config_production.yml | 9 ++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/_config.yml b/_config.yml index 45df16f9f7e8..357841ceed4a 100644 --- a/_config.yml +++ b/_config.yml @@ -16,7 +16,19 @@ incremental: true permalink: pretty safe: false lsi: false -exclude: ["_samples", "_scripts", "404.html", "datacenter", "ee", "index.html", "js/metadata.json"] + +exclude: + - _samples + - _scripts + - 404.html + - datacenter + - docker-bake.hcl + - ee + - index.html + - js/metadata.json + - LICENSE + - Makefile + - README.md # Component versions -- address like site.docker_ce_version # You can't have - characters in these for non-YAML reasons diff --git a/_config_production.yml b/_config_production.yml index cb30780092f0..244f04127cdb 100644 --- a/_config_production.yml +++ b/_config_production.yml @@ -5,7 +5,14 @@ # Override the exclusion list to include files that are excluded in "development", # such as the "enterprise" stubs, which are in place to facilitate redirects # to Mirantis. -exclude: ["_scripts", "404.html", "index.html"] +exclude: + - _scripts + - 404.html + - docker-bake.hcl + - index.html + - LICENSE + - Makefile + - README.md # Google Analytics, etc. google_analytics: GTM-WL2QLG5