jekyll: fix sitemap lastmod

.dockerignore

@@ -1,5 +1,4 @@
 .DS_Store
-.git
 .github
 .gitignore
 .idea

Dockerfile

@@ -47,6 +47,7 @@ ARG JEKYLL_ENV
 ARG DOCS_URL
 ENV TARGET=/out
 RUN --mount=type=bind,target=.,rw \
+    --mount=type=cache,target=/tmp/docker-docs-clone \
     --mount=type=cache,target=/src/.jekyll-cache <<EOT
   set -eu
   CONFIG_FILES=_config.yml$([ "$JEKYLL_ENV" = "production" ] && echo ",_config_production.yml" || true)

Gemfile

@@ -15,8 +15,8 @@ end
 #  more info: https://github.com/docker/docs/issues/14788
 gem 'rouge', '3.27.0'
 
-gem 'archive-zip', '0.12.0'
 gem 'front_matter_parser', '1.0.1'
+gem 'git', '1.12.0'
 gem 'html-proofer', '3.19.4'
 gem 'mdl', '0.11.0'
 gem 'octopress-hooks', '2.6.2'

Gemfile.lock

@@ -1,11 +1,9 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
-    archive-zip (0.12.0)
-      io-like (~> 0.3.0)
-    chef-utils (17.10.0)
+    addressable (2.8.1)
+      public_suffix (>= 2.0.2, < 6.0)
+    chef-utils (18.0.161)
       concurrent-ruby
     colorator (1.1.0)
     concurrent-ruby (1.1.10)
@@ -18,6 +16,9 @@ GEM
     ffi (1.15.5)
     forwardable-extended (2.6.0)
     front_matter_parser (1.0.1)
+    git (1.12.0)
+      addressable (~> 2.8)
+      rchardet (~> 1.8)
     html-proofer (3.19.4)
       addressable (~> 2.3)
       mercenary (~> 0.3)
@@ -29,7 +30,6 @@ GEM
     http_parser.rb (0.8.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    io-like (0.3.1)
     jekyll (4.2.2)
       addressable (~> 2.4)
       colorator (~> 1.0)
@@ -88,13 +88,14 @@ GEM
     parallel (1.22.1)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
-    public_suffix (4.0.7)
+    public_suffix (5.0.0)
     racc (1.6.0)
     rainbow (3.1.1)
     rake (13.0.6)
-    rb-fsevent (0.11.1)
+    rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
+    rchardet (1.8.0)
     rexml (3.2.5)
     rouge (3.27.0)
     safe_yaml (1.0.5)
@@ -114,8 +115,8 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  archive-zip (= 0.12.0)
   front_matter_parser (= 1.0.1)
+  git (= 1.12.0)
   html-proofer (= 3.19.4)
   jekyll (= 4.2.2)
   jekyll-redirect-from

_config.yml

@@ -102,6 +102,24 @@ defaults:
       toc_min: 2
       toc_max: 4
 
+  # Exclude from sitemap
+  - scope:
+      path: "assets/**"
+    values:
+      sitemap: false
+  - scope:
+      path: "**/nav.html"
+    values:
+      sitemap: false
+  - scope:
+      path: "google*.html"
+    values:
+      sitemap: false
+  - scope:
+      path: "**/*.pdf"
+    values:
+      sitemap: false
+
   # Set the correct edit-URL for some local and remote resources. We usually don't create a direct
   # edit link for these, and instead point to the directory that contains the file.
   - scope:

_plugins/fetch_remote.rb

@@ -1,9 +1,8 @@
-require 'archive/zip'
 require 'front_matter_parser'
+require 'git'
 require 'jekyll'
 require 'json'
 require 'octopress-hooks'
-require 'open-uri'
 require 'rake'
 
 require_relative 'util.rb'
@@ -12,14 +11,6 @@ module Jekyll
   class FetchRemote < Octopress::Hooks::Site
     priority :highest
 
-    def self.download(url, dest)
-      uri = URI.parse(url)
-      result = File.join(dest, File.basename(uri.path))
-      puts "    Downloading #{url}"
-      IO.copy_stream(URI.open(url), result)
-      return result
-    end
-
     def self.copy(src, dest)
       if (tmp = Array.try_convert(src))
         tmp.each do |s|
@@ -50,75 +41,87 @@ def pre_read(site)
       beginning_time = Time.now
       puts "Starting plugin fetch_remote.rb..."
       site.config['fetch-remote'].each do |entry|
-        puts "  Repo #{entry['repo']} (#{entry['ref']})"
-        Dir.mktmpdir do |tmpdir|
-          tmpfile = FetchRemote.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir)
-          Dir.mktmpdir do |ztmpdir|
-            puts "    Extracting #{tmpfile}"
-            Archive::Zip.extract(
-              tmpfile,
-              ztmpdir,
-              :create => true
-            )
-            entry['paths'].each do |path|
-              if File.extname(path['dest']) != ""
-                if path['src'].size > 1
-                  raise "Cannot use file destination #{path['dest']} with multiple sources"
-                end
-                FileUtils.mkdir_p File.dirname(path['dest'])
-              else
-                FileUtils.mkdir_p path['dest']
-              end
+        puts "  Repo #{entry['repo']}"
 
-              puts "    Copying files"
+        gituri = Git::URL.parse(entry['repo'])
+        clonedir = "#{Dir.tmpdir}/docker-docs-clone#{gituri.path}"
+        if Dir.exist?(clonedir)
+          puts "    Opening #{clonedir}"
+          begin
+            git = Git.open(clonedir)
+            puts "    Fetching #{entry['ref']}"
+            git.fetch
+            git.checkout(entry['ref'])
+          rescue => e
+            FileUtils.rm_rf(clonedir)
+            puts "    Cloning repository into #{clonedir}"
+            git = Git.clone("#{entry['repo']}.git", Pathname.new(clonedir), branch: entry['ref'], depth: 1)
+          end
+        else
+          puts "    Cloning repository into #{clonedir}"
+          git = Git.clone("#{entry['repo']}.git", Pathname.new(clonedir), branch: entry['ref'], depth: 1)
+        end
 
-              # prepare file list to be copied
-              files = FileList[]
-              path['src'].each do |src|
-                if "#{src}".start_with?("!")
-                  files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!")))
-                else
-                  files.include(File.join(ztmpdir, "*/#{src}"))
-                end
-              end
+        entry['paths'].each do |path|
+          if File.extname(path['dest']) != ""
+            if path['src'].size > 1
+              raise "Cannot use file destination #{path['dest']} with multiple sources"
+            end
+            FileUtils.mkdir_p File.dirname(path['dest'])
+          else
+            FileUtils.mkdir_p path['dest']
+          end
 
-              files.each do |file|
-                FetchRemote.copy(file, path['dest']) do |s, d|
-                  s = File.realpath(s)
-                  # traverse source directory
-                  FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent|
-                    file_clean = ent.path.delete_prefix(ztmpdir).split("/").drop(2).join("/")
-                    destent = FileUtils::Entry_.new(d, ent.rel, false)
-                    puts "      #{file_clean} => #{destent.path}"
+          puts "    Copying files"
 
-                    if File.file?(destent.path)
-                      fmp = FrontMatterParser::Parser.parse_file(destent.path)
-                      if fmp['fetch_remote'].nil?
-                        raise "Local file #{destent.path} already exists"
-                      end
-                      line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil)
-                      lines = File.readlines(ent.path)[line_start..line_end]
-                      File.open(destent.path, "a") { |fow| fow.puts lines.join }
-                    else
-                      ent.copy destent.path
-                    end
+          # prepare file list to be copied
+          files = FileList[]
+          path['src'].each do |src|
+            if "#{src}".start_with?("!")
+              files.exclude(File.join(clonedir, "/"+"#{src}".delete_prefix("!")))
+            else
+              files.include(File.join(clonedir, "/#{src}"))
+            end
+          end
 
-                    next unless File.file?(ent.path) && File.extname(ent.path) == ".md"
-                    # set edit and issue url and remote info for markdown files in site config defaults
-                    edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}"
-                    issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](#{get_docs_url}/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)"
-                    puts "        edit_url:  #{edit_url}"
-                    puts "        issue_url: #{issue_url}"
-                    site.config['defaults'] << {
-                      "scope" => { "path" => destent.path },
-                      "values" => {
-                        "edit_url" => edit_url,
-                        "issue_url" => issue_url
-                      },
-                    }
-                  end, proc do |_| end)
+          files.each do |file|
+            FetchRemote.copy(file, path['dest']) do |s, d|
+              s = File.realpath(s)
+              # traverse source directory
+              FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent|
+                file_clean = ent.path.delete_prefix(clonedir).split("/").drop(1).join("/")
+                destent = FileUtils::Entry_.new(d, ent.rel, false)
+                puts "      #{file_clean} => #{destent.path}"
+
+                if File.file?(destent.path)
+                  fmp = FrontMatterParser::Parser.parse_file(destent.path)
+                  if fmp['fetch_remote'].nil?
+                    raise "Local file #{destent.path} already exists"
+                  end
+                  line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil)
+                  lines = File.readlines(ent.path)[line_start..line_end]
+                  File.open(destent.path, "a") { |fow| fow.puts lines.join }
+                else
+                  ent.copy destent.path
                 end
-              end
+
+                next unless File.file?(ent.path) && File.extname(ent.path) == ".md"
+                # set edit and issue url and remote info for markdown files in site config defaults
+                edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}"
+                issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](#{get_docs_url}/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)"
+                last_modified_at = git.log.path(file_clean).first.date.strftime(LastModifiedAt::DATE_FORMAT)
+                puts "        edit_url:         #{edit_url}"
+                puts "        issue_url:        #{issue_url}"
+                puts "        last_modified_at: #{last_modified_at}"
+                site.config['defaults'] << {
+                  "scope" => { "path" => destent.path },
+                  "values" => {
+                    "edit_url" => edit_url,
+                    "issue_url" => issue_url,
+                    "last_modified_at" => last_modified_at,
+                  },
+                }
+              end, proc do |_| end)
             end
           end
         end

_plugins/last_modified_at.rb

@@ -0,0 +1,35 @@
+require 'git'
+require 'jekyll'
+require 'octopress-hooks'
+
+module Jekyll
+  class LastModifiedAt < Octopress::Hooks::Site
+    DATE_FORMAT = '%Y-%m-%d %H:%M:%S %z'
+    def pre_render(site)
+      if get_docs_url == "http://localhost:4000"
+        # Do not generate last_modified_at for local development
+        return
+      end
+
+      beginning_time = Time.now
+      Jekyll.logger.info "Starting plugin last_modified_at.rb..."
+
+      git = Git.open(site.source)
+      site.pages.each do |page|
+        next if page.relative_path == "redirect.html"
+        next unless File.extname(page.relative_path) == ".md" || File.extname(page.relative_path) == ".html"
+        unless page.data.key?('last_modified_at')
+          begin
+            page.data['last_modified_at'] = git.log.path(page.relative_path).first.date.strftime(DATE_FORMAT)
+          rescue => e
+            # Ignored
+          end
+        end
+        puts"  #{page.relative_path}\n    last_modified_at: #{page.data['last_modified_at']}"
+      end
+
+      end_time = Time.now
+      Jekyll.logger.info "done in #{(end_time - beginning_time)} seconds"
+    end
+  end
+end