jekyll: create plugin to fetch remote resources

Dockerfile

@@ -12,15 +12,10 @@ ARG BUNDLER_VERSION=2.3.13
 ARG JEKYLL_ENV=development
 ARG DOMAIN=docs.docker.com
 
-ARG ENGINE_BRANCH="20.10"
-ARG DISTRIBUTION_BRANCH="release/2.7"
-ARG COMPOSE_CLI_BRANCH="main"
-ARG EXTENSIONS_SDK_BRANCH="main"
-
 # Base stage for building
 FROM ruby:${RUBY_VERSION}-alpine AS base
 WORKDIR /src
-RUN apk add --no-cache bash build-base git subversion wget
+RUN apk add --no-cache bash build-base git
 
 # Gem stage will install bundler used as dependency manager
 # for our dependencies in Gemfile for Jekyll
@@ -45,33 +40,15 @@ RUN bundle update \
 FROM scratch AS vendor
 COPY --from=vendored /out /
 
-# Fetch upstream resources (reference documentation)
-# Only add the files that are needed to build these reference docs, so that these
-# docs are only rebuilt if changes were made to ENGINE_BRANCH or DISTRIBUTION_BRANCH.
-FROM base AS upstream-resources
-WORKDIR /out
-COPY ./_scripts/fetch-upstream-resources.sh ./_scripts/
-ARG ENGINE_BRANCH
-ARG DISTRIBUTION_BRANCH
-ARG COMPOSE_CLI_BRANCH
-ARG EXTENSIONS_SDK_BRANCH
-RUN ./_scripts/fetch-upstream-resources.sh .
-
 # Build the static HTML for the current docs.
 # After building with jekyll, fix up some links
 FROM gem AS generate
 ARG JEKYLL_ENV
 ARG DOMAIN
 ENV TARGET=/out
-COPY . .
-COPY --from=upstream-resources /out .
-RUN --mount=type=cache,target=/src/.jekyll-cache <<EOT
+RUN --mount=type=bind,target=.,rw \
+  --mount=type=cache,target=/src/.jekyll-cache <<EOT
 set -eu
-
-# substitute the "{site.latest_engine_api_version}" in the title for the latest
-# API docs, based on the latest_engine_api_version parameter in _config.yml
-(set -x ; ./_scripts/update-api-toc.sh)
-
 if [ "${JEKYLL_ENV}" = "production" ]; then
   (
     set -x
@@ -86,7 +63,6 @@ else
     echo '[]' > ${TARGET}/js/metadata.json
   )
 fi
-
 find ${TARGET} -type f -name '*.html' | while read i; do
   sed -i 's#\(<a[^>]* href="\)https://${DOMAIN}/#\1/#g' "$i"
 done

Gemfile

@@ -2,8 +2,13 @@ source 'https://rubygems.org'
 
 gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]
 gem 'wdm', '>= 0.1.0' if Gem.win_platform?
+
 gem 'jekyll'
 gem 'jekyll-redirect-from'
 gem 'jekyll-relative-links'
 gem 'jekyll-sitemap'
+
+gem 'archive-zip'
 gem 'html-proofer'
+gem 'octopress-hooks'
+gem 'rake'

Gemfile.lock

@@ -3,6 +3,8 @@ GEM
   specs:
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
+    archive-zip (0.12.0)
+      io-like (~> 0.3.0)
     colorator (1.1.0)
     concurrent-ruby (1.1.10)
     em-websocket (0.5.3)
@@ -24,6 +26,7 @@ GEM
     http_parser.rb (0.8.0)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
+    io-like (0.3.1)
     jekyll (4.2.2)
       addressable (~> 2.4)
       colorator (~> 1.0)
@@ -58,14 +61,17 @@ GEM
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     mercenary (0.4.0)
-    nokogiri (1.13.5-x86_64-linux)
+    nokogiri (1.13.6-x86_64-linux)
       racc (~> 1.4)
+    octopress-hooks (2.6.2)
+      jekyll (>= 2.0)
     parallel (1.22.1)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.7)
     racc (1.6.0)
     rainbow (3.1.1)
+    rake (13.0.6)
     rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
@@ -87,11 +93,14 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  archive-zip
   html-proofer
   jekyll
   jekyll-redirect-from
   jekyll-relative-links
   jekyll-sitemap
+  octopress-hooks
+  rake
   tzinfo-data
 
 BUNDLED WITH

_config.yml

@@ -104,3 +104,62 @@ defaults:
       path: engine/reference/commandline
     values:
       skip_read_time: true
+
+# Fetch upstream resources (reference documentation) used by _plugins/fetch_remote.rb
+# - repo is the GitHub repository to fetch from
+# - ref the Git reference
+# - paths is a list to the resources within the remote repository
+#   - dest is the destination path within the working tree
+#   - src is a list of glob source paths within the remote repository
+fetch-remote:
+  - repo: "https://github.com/docker/cli"
+    ref: "20.10"
+    paths:
+      - dest: "engine/extend"
+        src:
+          - "docs/extend/**"
+      - dest: "engine"
+        src:
+          - "docs/deprecated.md"
+      - dest: "engine/reference"
+        src:
+          - "docs/reference/builder.md"
+          - "docs/reference/run.md"
+      - dest: "engine/reference/commandline"
+        src:
+          - "docs/reference/commandline/cli.md"
+          - "docs/reference/commandline/dockerd.md"
+
+  - repo: "https://github.com/docker/docker"
+    ref: "20.10"
+    paths:
+      - dest: "engine/api"
+        src:
+          - "docs/api/**"
+
+  - repo: "https://github.com/docker/compose-cli"
+    ref: "main"
+    paths:
+      - dest: "cloud"
+        src:
+          - "docs/*.md"
+          - "!docs/README.md" # readme to make things nice in the compose-cli repo, but meaningless here
+          - "!docs/architecture.md" # Compose-CLI architecture, unrelated to cloud integration
+
+  - repo: "https://github.com/docker/extensions-sdk"
+    ref: "main"
+    paths:
+      - dest: "desktop/extensions-sdk"
+        src:
+          - "docs/**"
+
+  - repo: "https://github.com/distribution/distribution"
+    ref: "release/2.7"
+    paths:
+      - dest: "registry/spec"
+        src:
+          - "docs/spec/**"
+          - "!docs/spec/api.md.tmpl"
+      - dest: "registry"
+        src:
+          - "docs/configuration.md"

_plugins/fetch_remote.rb

@@ -0,0 +1,64 @@
+require 'archive/zip'
+require 'jekyll'
+require 'json'
+require 'octopress-hooks'
+require 'open-uri'
+require 'rake'
+
+module Jekyll
+  def self.download(url, dest)
+    uri = URI.parse(url)
+    result = File.join(dest, File.basename(uri.path))
+    Jekyll.logger.info "    Downloading #{url}"
+    IO.copy_stream(URI.open(url), result)
+    return result
+  end
+
+  class FetchRemote < Octopress::Hooks::Site
+    def pre_read(site)
+      beginning_time = Time.now
+      Jekyll.logger.info "Starting plugin fetch_remote.rb..."
+      site.config['fetch-remote'].each do |entry|
+        Jekyll.logger.info "  Repo #{entry['repo']} (#{entry['ref']})"
+        Dir.mktmpdir do |tmpdir|
+          tmpfile = Jekyll.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir)
+          Dir.mktmpdir do |ztmpdir|
+            Jekyll.logger.info "    Extracting #{tmpfile}"
+            Archive::Zip.extract(
+              tmpfile,
+              ztmpdir,
+              :create => true
+            )
+            entry['paths'].each do |path|
+              Jekyll.logger.info "    Copying files to ./#{path['dest']}/"
+              files = FileList[]
+              path['src'].each do |src|
+                if "#{src}".start_with?("!")
+                  files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!")))
+                else
+                  files.include(File.join(ztmpdir, "*/#{src}"))
+                end
+              end
+              files.each do |file|
+                Jekyll.logger.info "      #{file.delete_prefix(ztmpdir)}"
+              end
+              FileUtils.mkdir_p path['dest']
+              FileUtils.cp_r(files, path['dest'])
+            end
+          end
+        end
+      end
+
+      Jekyll.logger.info "  Fixing up URLs in swagger files"
+      Dir.glob("./engine/api/*.yaml") do |file_name|
+        Jekyll.logger.info "    #{file_name}"
+        text = File.read(file_name)
+        replace = text.gsub!("https://docs.docker.com/", "")
+        File.open(file_name, "w") { |file| file.puts replace }
+      end
+
+      end_time = Time.now
+      Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds"
+    end
+  end
+end

_plugins/update_api_toc.rb

@@ -0,0 +1,26 @@
+require 'jekyll'
+require 'octopress-hooks'
+
+module Jekyll
+
+  class UpdateApiToc < Octopress::Hooks::Site
+    def pre_read(site)
+      beginning_time = Time.now
+      Jekyll.logger.info "Starting plugin update_api_toc.rb..."
+
+      if File.file?("_config.yml") && File.file?("_data/toc.yaml")
+        # substitute the "{site.latest_engine_api_version}" in the title for the latest
+        # API docs, based on the latest_engine_api_version parameter in _config.yml
+        engine_ver = site.config['latest_engine_api_version']
+        toc_file = File.read("_data/toc.yaml")
+        replace = toc_file.gsub!("{{ site.latest_engine_api_version }}", engine_ver)
+        Jekyll.logger.info "  Replacing '{{ site.latest_engine_api_version }}' with #{engine_ver} in _data/toc.yaml"
+        File.open("_data/toc.yaml", "w") { |file| file.puts replace }
+      end
+
+      end_time = Time.now
+      Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds"
+    end
+  end
+
+end

docker-bake.hcl

@@ -11,7 +11,7 @@ target "release" {
   args = {
     JEKYLL_ENV = JEKYLL_ENV
   }
-  no-cache-filter = ["upstream-resources"]
+  no-cache-filter = ["generate"]
   output = ["./_site"]
 }