From dd64314a6d932c00af53a7c1fdd405cb01c348a6 Mon Sep 17 00:00:00 2001 From: Michael Irwin Date: Wed, 27 Jul 2022 15:22:23 -0400 Subject: [PATCH 1/5] Update plugins to not make file changes if no changes are made --- _plugins/fix_urls.rb | 8 ++++++-- _plugins/update_api_toc.rb | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/_plugins/fix_urls.rb b/_plugins/fix_urls.rb index 1d96f38ebc65..8df274e5142a 100644 --- a/_plugins/fix_urls.rb +++ b/_plugins/fix_urls.rb @@ -9,10 +9,14 @@ def post_read(site) Jekyll.logger.info " Fixing up URLs in swagger files" Dir.glob(%w[./docker-hub/api/*.yaml ./engine/api/*.yaml]) do |file_name| - Jekyll.logger.info " #{file_name}" text = File.read(file_name) replace = text.gsub("https://docs.docker.com", "") - File.open(file_name, "w") { |file| file.puts replace } + if text == replace + Jekyll.logger.info " #{file_name} (skipped - no change)" + else + Jekyll.logger.info " #{file_name}" + File.open(file_name, "w") { |file| file.puts replace } + end end end_time = Time.now diff --git a/_plugins/update_api_toc.rb b/_plugins/update_api_toc.rb index b8e6170a3cee..e2ffe5cfb2cf 100644 --- a/_plugins/update_api_toc.rb +++ b/_plugins/update_api_toc.rb @@ -13,8 +13,10 @@ def pre_read(site) engine_ver = site.config['latest_engine_api_version'] toc_file = File.read("_data/toc.yaml") replace = toc_file.gsub!("{{ site.latest_engine_api_version }}", engine_ver) - Jekyll.logger.info " Replacing '{{ site.latest_engine_api_version }}' with #{engine_ver} in _data/toc.yaml" - File.open("_data/toc.yaml", "w") { |file| file.puts replace } + if toc_file != replace + Jekyll.logger.info " Replacing '{{ site.latest_engine_api_version }}' with #{engine_ver} in _data/toc.yaml" + File.open("_data/toc.yaml", "w") { |file| file.puts replace } + end end end_time = Time.now From 75b16ad4268d6a2ccc8a5b4964de1e4c2239e4a5 Mon Sep 17 00:00:00 2001 From: Michael Irwin Date: Wed, 27 Jul 2022 15:24:24 -0400 Subject: [PATCH 2/5] Add completion_marker support to fetch_remote plugin The intent is to support jekyll serve without redownloading all packages every time a change is made. With this, if the marker file exists, no secondary download will occur. --- _config.yml | 7 +++ _plugins/fetch_remote.rb | 122 ++++++++++++++++++++------------------- 2 files changed, 70 insertions(+), 59 deletions(-) diff --git a/_config.yml b/_config.yml index 5239a2e03389..3141726509a5 100644 --- a/_config.yml +++ b/_config.yml @@ -123,6 +123,7 @@ defaults: # Fetch upstream resources (reference documentation) used by _plugins/fetch_remote.rb # - repo is the GitHub repository to fetch from # - ref the Git reference +# - completion_marker is a file path that, if it exists, will prevent a redownload from occurring # - paths is a list to the resources within the remote repository # - dest is the destination path within the working tree # - src is a list of glob source paths within the remote repository @@ -130,6 +131,7 @@ fetch-remote: - repo: "https://github.com/docker/cli" default_branch: "master" ref: "20.10" + completion_marker: "engine/deprecated.md" paths: - dest: "engine/extend" src: @@ -148,6 +150,7 @@ fetch-remote: - repo: "https://github.com/docker/docker" default_branch: "master" ref: "20.10" + completion_marker: "engine/api/version-history.md" paths: - dest: "engine/api" src: @@ -156,6 +159,7 @@ fetch-remote: - repo: "https://github.com/docker/compose-cli" default_branch: "main" ref: "main" + completion_marker: "cloud/ecs-integration.md" paths: - dest: "cloud" src: @@ -166,6 +170,7 @@ fetch-remote: - repo: "https://github.com/docker/buildx" default_branch: "master" ref: "master" + completion_marker: "build/bake/index.md" paths: - dest: "build/bake" src: @@ -174,6 +179,7 @@ fetch-remote: - repo: "https://github.com/distribution/distribution" default_branch: "main" ref: "release/2.7" + completion_marker: "registry/spec/index.md" paths: - dest: "registry/spec" src: @@ -186,6 +192,7 @@ fetch-remote: - repo: "https://github.com/moby/buildkit" default_branch: "master" ref: "master" + completion_marker: "engine/reference/builder.md" paths: - dest: "engine/reference/builder.md" src: diff --git a/_plugins/fetch_remote.rb b/_plugins/fetch_remote.rb index 8513fc509980..103d390791eb 100644 --- a/_plugins/fetch_remote.rb +++ b/_plugins/fetch_remote.rb @@ -49,79 +49,83 @@ def pre_read(site) puts "Starting plugin fetch_remote.rb..." site.config['fetch-remote'].each do |entry| puts " Repo #{entry['repo']} (#{entry['ref']})" - Dir.mktmpdir do |tmpdir| - tmpfile = FetchRemote.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir) - Dir.mktmpdir do |ztmpdir| - puts " Extracting #{tmpfile}" - Archive::Zip.extract( - tmpfile, - ztmpdir, - :create => true - ) - entry['paths'].each do |path| - if File.extname(path['dest']) != "" - if path['src'].size > 1 - raise "Cannot use file destination #{path['dest']} with multiple sources" + + if(File.exist?(entry['completion_marker'])) + puts " Skipping fetch as #{entry['completion_marker']} exists" + else + Dir.mktmpdir do |tmpdir| + tmpfile = FetchRemote.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir) + Dir.mktmpdir do |ztmpdir| + puts " Extracting #{tmpfile}" + Archive::Zip.extract( + tmpfile, + ztmpdir, + :create => true + ) + entry['paths'].each do |path| + if File.extname(path['dest']) != "" + if path['src'].size > 1 + raise "Cannot use file destination #{path['dest']} with multiple sources" + end + FileUtils.mkdir_p File.dirname(path['dest']) + else + FileUtils.mkdir_p path['dest'] end - FileUtils.mkdir_p File.dirname(path['dest']) - else - FileUtils.mkdir_p path['dest'] - end - puts " Copying files" + puts " Copying files" - # prepare file list to be copied - files = FileList[] - path['src'].each do |src| - if "#{src}".start_with?("!") - files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!"))) - else - files.include(File.join(ztmpdir, "*/#{src}")) + # prepare file list to be copied + files = FileList[] + path['src'].each do |src| + if "#{src}".start_with?("!") + files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!"))) + else + files.include(File.join(ztmpdir, "*/#{src}")) + end end - end - files.each do |file| - FetchRemote.copy(file, path['dest']) do |s, d| - s = File.realpath(s) - # traverse source directory - FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent| - file_clean = ent.path.delete_prefix(ztmpdir).split("/").drop(2).join("/") - destent = FileUtils::Entry_.new(d, ent.rel, false) - puts " #{file_clean} => #{destent.path}" + files.each do |file| + FetchRemote.copy(file, path['dest']) do |s, d| + s = File.realpath(s) + # traverse source directory + FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent| + file_clean = ent.path.delete_prefix(ztmpdir).split("/").drop(2).join("/") + destent = FileUtils::Entry_.new(d, ent.rel, false) + puts " #{file_clean} => #{destent.path}" - if File.file?(destent.path) - fmp = FrontMatterParser::Parser.parse_file(destent.path) - if fmp['fetch_remote'].nil? - raise "Local file #{destent.path} already exists" + if File.file?(destent.path) + fmp = FrontMatterParser::Parser.parse_file(destent.path) + if fmp['fetch_remote'].nil? + raise "Local file #{destent.path} already exists" + end + line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil) + lines = File.readlines(ent.path)[line_start..line_end] + File.open(destent.path, "a") { |fow| fow.puts lines.join } + else + ent.copy destent.path end - line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil) - lines = File.readlines(ent.path)[line_start..line_end] - File.open(destent.path, "a") { |fow| fow.puts lines.join } - else - ent.copy destent.path - end - next unless File.file?(ent.path) && File.extname(ent.path) == ".md" - # set edit and issue url and remote info for markdown files in site config defaults - edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}" - issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](https://docs.docker.com/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)" - puts " edit_url: #{edit_url}" - puts " issue_url: #{issue_url}" - site.config['defaults'] << { - "scope" => { "path" => destent.path }, - "values" => { - "edit_url" => edit_url, - "issue_url" => issue_url - }, - } - end, proc do |_| end) + next unless File.file?(ent.path) && File.extname(ent.path) == ".md" + # set edit and issue url and remote info for markdown files in site config defaults + edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}" + issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](https://docs.docker.com/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)" + puts " edit_url: #{edit_url}" + puts " issue_url: #{issue_url}" + site.config['defaults'] << { + "scope" => { "path" => destent.path }, + "values" => { + "edit_url" => edit_url, + "issue_url" => issue_url + }, + } + end, proc do |_| end) + end end end end end end end - end_time = Time.now puts "done in #{(end_time - beginning_time)} seconds" end From 2a37f34b549520fa523ea6addc7ce15cf3e7fbee Mon Sep 17 00:00:00 2001 From: Michael Irwin Date: Wed, 27 Jul 2022 15:25:32 -0400 Subject: [PATCH 3/5] Update compose file to use jekyll serve with livereload support Now, changes will be seen much faster, removing the requirement to rebuild and restart the container every time you want to see a change --- Dockerfile | 5 +++++ dev-start.sh | 8 ++++++++ docker-compose.yml | 6 +++++- 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100755 dev-start.sh diff --git a/Dockerfile b/Dockerfile index b409c4407d64..1517b5c53d0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,11 @@ RUN bundle update \ FROM scratch AS vendor COPY --from=vendored /out / +# Serve the files using jekyll to enable file watching +# and livereload for a faster inner loop +FROM gem AS dev +CMD ["./dev-start.sh"] + # Build the static HTML for the current docs. # After building with jekyll, fix up some links FROM gem AS generate diff --git a/dev-start.sh b/dev-start.sh new file mode 100755 index 000000000000..a2263805818d --- /dev/null +++ b/dev-start.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ "$JEKYLL_ENV" == "production" ]; then + echo "Starting in production mode" + exec bundle exec jekyll serve --host=0.0.0.0 -l --config _config.yml,_config_production.yml +else + exec bundle exec jekyll serve --host=0.0.0.0 -l +fi diff --git a/docker-compose.yml b/docker-compose.yml index 7d5ab9d730b5..8d66b1a9f7c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,10 @@ services: args: - JEKYLL_ENV context: . + target: dev image: docs/docstage ports: - - "4000:4000" + - 4000:4000 + - 35729:35729 + volumes: + - ./:/src From 8643309df84850bf0b652b964aaae2e0d0fac4ce Mon Sep 17 00:00:00 2001 From: Michael Irwin Date: Wed, 27 Jul 2022 15:48:02 -0400 Subject: [PATCH 4/5] Fix completion_marker used for cloud integration --- _config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_config.yml b/_config.yml index 3141726509a5..36f70def7e4d 100644 --- a/_config.yml +++ b/_config.yml @@ -159,7 +159,7 @@ fetch-remote: - repo: "https://github.com/docker/compose-cli" default_branch: "main" ref: "main" - completion_marker: "cloud/ecs-integration.md" + completion_marker: "cloud/ecs-compose-features.md" paths: - dest: "cloud" src: From b66c509bd35dc593fde176d391e0d87f27224960 Mon Sep 17 00:00:00 2001 From: Michael Irwin Date: Wed, 27 Jul 2022 15:59:43 -0400 Subject: [PATCH 5/5] Handle case in which fetch_remote is using a file with frontmatter --- _plugins/fetch_remote.rb | 126 ++++++++++++++++++++------------------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/_plugins/fetch_remote.rb b/_plugins/fetch_remote.rb index 103d390791eb..12556b457f5d 100644 --- a/_plugins/fetch_remote.rb +++ b/_plugins/fetch_remote.rb @@ -51,75 +51,79 @@ def pre_read(site) puts " Repo #{entry['repo']} (#{entry['ref']})" if(File.exist?(entry['completion_marker'])) - puts " Skipping fetch as #{entry['completion_marker']} exists" - else - Dir.mktmpdir do |tmpdir| - tmpfile = FetchRemote.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir) - Dir.mktmpdir do |ztmpdir| - puts " Extracting #{tmpfile}" - Archive::Zip.extract( - tmpfile, - ztmpdir, - :create => true - ) - entry['paths'].each do |path| - if File.extname(path['dest']) != "" - if path['src'].size > 1 - raise "Cannot use file destination #{path['dest']} with multiple sources" - end - FileUtils.mkdir_p File.dirname(path['dest']) - else - FileUtils.mkdir_p path['dest'] + fmp = FrontMatterParser::Parser.parse_file(entry['completion_marker']) + if fmp.content != "" + puts " Skipping fetch as #{entry['completion_marker']} exists" + next + end + end + + Dir.mktmpdir do |tmpdir| + tmpfile = FetchRemote.download("#{entry['repo']}/archive/#{entry['ref']}.zip", tmpdir) + Dir.mktmpdir do |ztmpdir| + puts " Extracting #{tmpfile}" + Archive::Zip.extract( + tmpfile, + ztmpdir, + :create => true + ) + entry['paths'].each do |path| + if File.extname(path['dest']) != "" + if path['src'].size > 1 + raise "Cannot use file destination #{path['dest']} with multiple sources" end + FileUtils.mkdir_p File.dirname(path['dest']) + else + FileUtils.mkdir_p path['dest'] + end - puts " Copying files" + puts " Copying files" - # prepare file list to be copied - files = FileList[] - path['src'].each do |src| - if "#{src}".start_with?("!") - files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!"))) - else - files.include(File.join(ztmpdir, "*/#{src}")) - end + # prepare file list to be copied + files = FileList[] + path['src'].each do |src| + if "#{src}".start_with?("!") + files.exclude(File.join(ztmpdir, "*/"+"#{src}".delete_prefix("!"))) + else + files.include(File.join(ztmpdir, "*/#{src}")) end + end - files.each do |file| - FetchRemote.copy(file, path['dest']) do |s, d| - s = File.realpath(s) - # traverse source directory - FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent| - file_clean = ent.path.delete_prefix(ztmpdir).split("/").drop(2).join("/") - destent = FileUtils::Entry_.new(d, ent.rel, false) - puts " #{file_clean} => #{destent.path}" + files.each do |file| + FetchRemote.copy(file, path['dest']) do |s, d| + s = File.realpath(s) + # traverse source directory + FileUtils::Entry_.new(s, nil, false).wrap_traverse(proc do |ent| + file_clean = ent.path.delete_prefix(ztmpdir).split("/").drop(2).join("/") + destent = FileUtils::Entry_.new(d, ent.rel, false) + puts " #{file_clean} => #{destent.path}" - if File.file?(destent.path) - fmp = FrontMatterParser::Parser.parse_file(destent.path) - if fmp['fetch_remote'].nil? - raise "Local file #{destent.path} already exists" - end - line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil) - lines = File.readlines(ent.path)[line_start..line_end] - File.open(destent.path, "a") { |fow| fow.puts lines.join } - else - ent.copy destent.path + if File.file?(destent.path) + fmp = FrontMatterParser::Parser.parse_file(destent.path) + if fmp['fetch_remote'].nil? + raise "Local file #{destent.path} already exists" end + line_start, line_end = FetchRemote.resolve_line_numbers(fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_start'] : nil, fmp['fetch_remote'].kind_of?(Hash) ? fmp['fetch_remote']['line_end'] : nil) + lines = File.readlines(ent.path)[line_start..line_end] + File.open(destent.path, "a") { |fow| fow.puts lines.join } + else + ent.copy destent.path + end - next unless File.file?(ent.path) && File.extname(ent.path) == ".md" - # set edit and issue url and remote info for markdown files in site config defaults - edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}" - issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](https://docs.docker.com/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)" - puts " edit_url: #{edit_url}" - puts " issue_url: #{issue_url}" - site.config['defaults'] << { - "scope" => { "path" => destent.path }, - "values" => { - "edit_url" => edit_url, - "issue_url" => issue_url - }, - } - end, proc do |_| end) - end + next unless File.file?(ent.path) && File.extname(ent.path) == ".md" + # set edit and issue url and remote info for markdown files in site config defaults + edit_url = "#{entry['repo']}/edit/#{entry['default_branch']}/#{file_clean}" + issue_url = "#{entry['repo']}/issues/new?body=File: [#{file_clean}](https://docs.docker.com/#{destent.path.sub(/#{File.extname(destent.path)}$/, '')}/)" + puts " edit_url: #{edit_url}" + puts " issue_url: #{issue_url}" + site.config['defaults'] << { + "scope" => { "path" => destent.path }, + "values" => { + "edit_url" => edit_url, + "issue_url" => issue_url + }, + } + end, proc do |_| end) end end end