From a1621ffabb2320f323352a3c3384c57bc24a9544 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:41:51 -0500 Subject: [PATCH 01/10] support LANGUAGE and VERSION env vars as options --- script/sync-algolia-search-indices.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/sync-algolia-search-indices.js b/script/sync-algolia-search-indices.js index 586eb5cfdd91..105e0defa3fe 100755 --- a/script/sync-algolia-search-indices.js +++ b/script/sync-algolia-search-indices.js @@ -14,7 +14,9 @@ main() async function main () { const sync = require('../lib/algolia/sync') const opts = { - dryRun: 'DRY_RUN' in process.env + dryRun: 'DRY_RUN' in process.env, + language: process.env.LANGUAGE, + version: process.env.VERSION } await sync(opts) } From 180c7c7db1dd5d12b9a1f3fabed8c2f1058fc4de Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:43:39 -0500 Subject: [PATCH 02/10] build specified language or version indices only --- lib/algolia/sync.js | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/lib/algolia/sync.js b/lib/algolia/sync.js index fb46cd17cdd0..087b87cf0577 100644 --- a/lib/algolia/sync.js +++ b/lib/algolia/sync.js @@ -15,17 +15,43 @@ const allVersions = require('../all-versions') // e.g. `github-docs-dotcom-en.json` and `github-docs-2.14-ja.json` module.exports = async function syncAlgoliaIndices (opts = {}) { if (opts.dryRun) { - console.log('This is a dry run! The script will abort without uploading anything.') + console.log('This is a dry run! The script will build the indices locally but not upload anything.\n') rimraf(cacheDir) mkdirp(cacheDir) } + if (opts.language) { + if (!Object.keys(languages).includes(opts.language)) { + console.log(`Error! ${opts.language} not found. You must provide a currently supported two-letter language code.`) + process.exit(1) + } + } + + if (opts.version) { + if (!Object.keys(allVersions).includes(opts.version)) { + console.log(`Error! ${opts.version} not found. You must provide a currently supported version in format.`) + process.exit(1) + } + } + + // build indices for a specific language if provided; otherwise build indices for all languages + const languagesToBuild = opts.language + ? Object.keys(languages).filter(language => language === opts.language) + : Object.keys(languages) + + // build indices for a specific version if provided; otherwise build indices for all veersions + const versionsToBuild = opts.version + ? Object.keys(allVersions).filter(version => version === opts.version) + : Object.keys(allVersions) + + console.log(`Building indices for ${opts.language || 'all languages'} and ${opts.version || 'all versions'}.\n`) + // Exlude WIP pages, hidden pages, index pages, etc const indexablePages = await findIndexablePages() // Build and validate all indices - for (const languageCode in languages) { - for (const pageVersion in allVersions) { + for (const languageCode of languagesToBuild) { + for (const pageVersion of versionsToBuild) { // if GHES, resolves to the release number like 2.21, 2.22, etc. // if FPT, resolves to 'dotcom' // if GHAE, resolves to 'ghae' From 071879e97b9f01e6f030f39718139285087b028b Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:45:05 -0500 Subject: [PATCH 03/10] add new label-triggered workflow --- .../sync-single-english-algolia-index.yml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/sync-single-english-algolia-index.yml diff --git a/.github/workflows/sync-single-english-algolia-index.yml b/.github/workflows/sync-single-english-algolia-index.yml new file mode 100644 index 000000000000..bc2fb38ddc1e --- /dev/null +++ b/.github/workflows/sync-single-english-algolia-index.yml @@ -0,0 +1,44 @@ +name: Algolia Sync Single English Index + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + - unlocked + +# This workflow requires a label in the format `sync-english-index-for-` +jobs: + updateIndices: + name: Update English index for single version based on a label's version + if: github.repository == 'github/docs-internal' && github.event.label.name.startsWith('sync-english-index-for-') + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + - uses: actions/setup-node@56899e050abffc08c2b3b61f3ec6a79a9dc3223d + with: + node-version: 14.x + - name: cache node modules + uses: actions/cache@0781355a23dac32fd3bac414512f4b903437991a + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: npm ci + run: npm ci + - name: Get version from label + id: getVersion + - run: | + echo "::set-output name=version::$(github.event.label.name.split('sync-english-index-for-')[1])" + - name: Sync English index for single version + env: + VERSION: ${{ steps.getVersion.outputs.version }} + LANGUAGE: 'en' + ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }} + ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: npm run sync-search From 0a195e40eb1ed06617251adc89ab91e9c20b1b10 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:45:23 -0500 Subject: [PATCH 04/10] document the new command options and the new workflow --- contributing/search.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/contributing/search.md b/contributing/search.md index 35b240b678fe..1eee8fb17d22 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -26,6 +26,41 @@ To sync the indices from your development environment: 2. Run `npm run sync-search-dry-run`. This takes a while to complete. It will prepare, test, and validate all the indices without actually uploading anything to Algolia's servers. 3. Run `npm run sync-search` to prepare the indices again and upload them to the Algolia servers. +## Build indices for a specific language and/or version + +You can optionally build indices for a given language and/or a version. This is much faster than building all the indices. + +You can build them as a dry run, which creates the index locally but does not upload them to Algolia. For example: +``` +VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search-dry-run +``` +Or you can build and load them to Algolia to make them live: +``` +VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search +``` +The `VERSION` env var must be a currently supported version in `` format. The `LANGUAGE` env var must be a currently supported two-letter language code. + +### Label-triggered workflow + +For our daily shipping needs, the indices are synced to Algolia only after a PR is shipped to the default branch, which means the search data isn't available for up to an hour after the content goes live. + +But for GHES releases, we have additional needs. We want: + +1) The search results to be live immediately upon releasing the new content. +2) The search results to be previewable on the staging app the GHES release megabranch. + +Manually running `npm run sync` locally before shipping the megabranch can address #1, but it's cumbersome and hard to get the timing right. And it does not address #2. + +Now, docs team members can use an Actions workflow on GHES megabranch PRs by applying a label in this format: +``` +sync-english-index-for- +``` +This label will run a workflow on every push to **build and upload ONLY** the English index for the specified version. This means: + +* The GHES content will be searchable at the same time the megabranch is shipped, with no delay. +* The GHES content will be searchable on staging throughout content creation. +* No manual steps (unless anyone wants to do a dry-run test as described in the previous section). + ## Files - [.github/workflows/sync-algolia-search-indices.yml](.github/workflows/sync-algolia-search-indices.yml) - the GitHub Actions workflow file that updates search indices whenever the main branch is updated. From db79e8fc56b63780063f9a706e21ab95d5412f7a Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 10:59:18 -0500 Subject: [PATCH 05/10] Update contributing/search.md Co-authored-by: Kevin Heis --- contributing/search.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing/search.md b/contributing/search.md index 1eee8fb17d22..491a6919e3bd 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -38,7 +38,7 @@ Or you can build and load them to Algolia to make them live: ``` VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search ``` -The `VERSION` env var must be a currently supported version in `` format. The `LANGUAGE` env var must be a currently supported two-letter language code. +The `VERSION` environment variable must be a currently supported version in `` format. The `LANGUAGE` environment variable must be a currently supported two-letter language code. ### Label-triggered workflow From 67cc12fca165f312780f719519ed07c38f8235bb Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 11:02:02 -0500 Subject: [PATCH 06/10] remove extra hyphen --- .github/workflows/sync-single-english-algolia-index.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-single-english-algolia-index.yml b/.github/workflows/sync-single-english-algolia-index.yml index bc2fb38ddc1e..1ac65d8551ff 100644 --- a/.github/workflows/sync-single-english-algolia-index.yml +++ b/.github/workflows/sync-single-english-algolia-index.yml @@ -32,7 +32,7 @@ jobs: run: npm ci - name: Get version from label id: getVersion - - run: | + run: | echo "::set-output name=version::$(github.event.label.name.split('sync-english-index-for-')[1])" - name: Sync English index for single version env: From 6041ffcfeba004dda2f27854b14932c3803c24d1 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 11:18:10 -0500 Subject: [PATCH 07/10] Update contributing/search.md --- contributing/search.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing/search.md b/contributing/search.md index 491a6919e3bd..dabce854aa26 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -47,7 +47,7 @@ For our daily shipping needs, the indices are synced to Algolia only after a PR But for GHES releases, we have additional needs. We want: 1) The search results to be live immediately upon releasing the new content. -2) The search results to be previewable on the staging app the GHES release megabranch. +2) The search results to be viewable on the staging app for the GHES release megabranch. Manually running `npm run sync` locally before shipping the megabranch can address #1, but it's cumbersome and hard to get the timing right. And it does not address #2. From 1a8dedf4d50db56e69d5c6066c553e4a40b68b82 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 16:50:11 -0500 Subject: [PATCH 08/10] bunch of edits --- contributing/search.md | 92 ++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 34 deletions(-) diff --git a/contributing/search.md b/contributing/search.md index dabce854aa26..30919813b4c6 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -1,4 +1,18 @@ -## Search +# Search + +## Table of contents +- [Overview](#overview) +- [How it works](#how-it-works) +- [Manual sync from a checkout](#manual-sync-from-a-checkout) + - [Build without sync (dry run)](#build-without-sync-dry-run) + - [Build and sync](#build-and-sync) +- [Label-triggered workflow](#label-triggered-workflow) +- [Files](#files) + - [GitHub Actions workflow files](#github-actions-workflow-files) + - [Code files](#code-files) + + +## Overview This site's search functionality is powered by [Algolia](https://www.algolia.com), a third-party service. @@ -10,65 +24,75 @@ To see all existing search-related issues and pull requests, visit [github.com/g --- -## How it Works +## How it works The search data is synced automatically using a [GitHub Actions workflow](.github/workflows/sync-algolia-search-indices.yml) that is triggered by pushes to the `main` branch. This process generates structured data for all pages on the site, compares that data to what's currently on Algolia, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations. -The Actions workflow usually takes about five minutes, and the progress can be viewed (by GitHub employees) in the [Actions tab](https://github.com/github/docs/actions?query=workflow%3AAlgolia) of the repo. - -## Development +The Actions workflow progress can be viewed (by GitHub employees) in the [Actions tab](https://github.com/github/docs/actions?query=workflow%3AAlgolia) of the repo. -In cases where a publicity event like GitHub Satellite or GitHub Universe demands a very tight shipping window, it is also possible to manually sync the indices with Algolia's servers from your local checkout of the repo, before your feature branch is merged to main. Manually syncing the indices can also be useful to test an unreleased GitHub Enterprise version or a translated language (Portuguese, Chinese, etc) that is not yet in production. +## Manual sync from a checkout -To sync the indices from your development environment: +It is also possible to manually sync the indices to Algolia from your local checkout of the repo, before your branch is merged to `main`. -1. Make sure the two required environment variables `ALGOLIA_APPLICATION_ID` and `ALGOLIA_API_KEY` are set in your `.env` file. These can be retrieved from the [Algolia site](https://www.algolia.com/apps/ZI5KPY1HBE/api-keys/all). -2. Run `npm run sync-search-dry-run`. This takes a while to complete. It will prepare, test, and validate all the indices without actually uploading anything to Algolia's servers. -3. Run `npm run sync-search` to prepare the indices again and upload them to the Algolia servers. +**Prerequisite:** Make sure the environment variables `ALGOLIA_APPLICATION_ID` and `ALGOLIA_API_KEY` are set in your `.env` file. You can find these values on [Algolia](https://www.algolia.com/apps/ZI5KPY1HBE/api-keys/all). -## Build indices for a specific language and/or version +### Build without sync (dry run) -You can optionally build indices for a given language and/or a version. This is much faster than building all the indices. - -You can build them as a dry run, which creates the index locally but does not upload them to Algolia. For example: +To build all the indices without uploading them to Algolia's servers (this takes about an hour): ``` -VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search-dry-run +npm run sync-search-dry-run ``` -Or you can build and load them to Algolia to make them live: +To build indices for a specific language and/or version (this is much faster): ``` -VERSION='free-pro-team@latest' LANGUAGE='en' npm run sync-search +VERSION= LANGUAGE= npm run sync-search-dry-run ``` -The `VERSION` environment variable must be a currently supported version in `` format. The `LANGUAGE` environment variable must be a currently supported two-letter language code. +You can set `VERSION` and `LANGUAGE` individually, too. -### Label-triggered workflow +Substitute a currently supported version for `` and a currently supported two-letter language code for ``. -For our daily shipping needs, the indices are synced to Algolia only after a PR is shipped to the default branch, which means the search data isn't available for up to an hour after the content goes live. +### Build and sync -But for GHES releases, we have additional needs. We want: +To build all the indices and sync them to Algolia (this also takes about an hour): +``` +npm run sync-search +``` +To build indices for a specific language and/or version and sync them to Algolia: +``` +VERSION= npm run sync-search +``` +You can set `VERSION` and `LANGUAGE` individually, too. -1) The search results to be live immediately upon releasing the new content. -2) The search results to be viewable on the staging app for the GHES release megabranch. +Substitute a currently supported version for `` and a currently supported two-letter language code for ``. -Manually running `npm run sync` locally before shipping the megabranch can address #1, but it's cumbersome and hard to get the timing right. And it does not address #2. +## Label-triggered workflow -Now, docs team members can use an Actions workflow on GHES megabranch PRs by applying a label in this format: +Docs team members can use an Actions workflow on GHES release PRs by applying a label in this format: ``` sync-english-index-for- ``` -This label will run a workflow on every push to **build and upload ONLY** the English index for the specified version. This means: +This label will run a workflow on every push that **builds and uploads ONLY** the English index for the specified version. This means: -* The GHES content will be searchable at the same time the megabranch is shipped, with no delay. +* The GHES content will be searchable at the same time the release PR is shipped, with no delay. * The GHES content will be searchable on staging throughout content creation. -* No manual steps (unless anyone wants to do a dry-run test as described in the previous section). +* No manual steps (unless you want to do a [dry run test](#build-without-sync-dry-run)). + +Why do we need this? For our daily shipping needs, it's tolerable that search updates aren't available for up to an hour after the content goes live. But GHES releases are more time-sensitive, and writers have a greater need to preview search data on staging. ## Files -- [.github/workflows/sync-algolia-search-indices.yml](.github/workflows/sync-algolia-search-indices.yml) - the GitHub Actions workflow file that updates search indices whenever the main branch is updated. -- [javascripts/search.js](javascripts/search.js) - the browser-side code that enables search using Algolia's [InstantSearch.js](https://github.com/algolia/instantsearch.js/) library. -- [lib/algolia/client.js](lib/algolia/client.js) - a thin wrapper around the [algoliasearch](https://ghub.io/algoliasearch) Node.js module for interacting with the Algolia API. -- [lib/algolia/search-index.js](lib/algolia/search-index.js) - a class for generating structured search data from repository content and syncing it with the remote Algolia service. This class has built-in validation to ensure that all records are valid before they're uploaded. This class also takes care of removing deprecated records, and compares existing remote records with the latest local records to avoid uploading records that haven't changed. -- [script/sync-algolia-search-indices.js](script/sync-algolia-search-indices.js) - the script used by the Actions workflow to update search indices on our Algolia account. This can also be [run in the development environment](#development). -- [tests/algolia-search.js](tests/algolia-search.js) - tests! +### GitHub Actions workflow files + +- [`.github/workflows/sync-algolia-search-indices.yml`](.github/workflows/sync-algolia-search-indices.yml) - Builds and syncs search indices whenever the `main` branch is pushed to (that is, on production deploys). +- [`.github/workflows/dry-run-sync-algolia-search-indices.yml`](.github/workflows/dry-run-sync-algolia-search-indices.yml) - This workflow can be run manually (via `workflow_dispatch`) to do a dry run build of all the indices. Useful for confirming that the indices can build without erroring out. +- [`.github/workflows/sync-single-english-algolia-index.yml`](.github/workflows/sync-single-english-algolia-index.yml) - This workflow is run when a label in the right format is applied to a PR. See "[Label-triggered workflow](#label-triggered-workflow)" for details. + +### Code files + +- [javascripts/search.js](javascripts/search.js) - The browser-side code that enables search using Algolia's [InstantSearch.js](https://github.com/algolia/instantsearch.js/) library. +- [lib/algolia/client.js](lib/algolia/client.js) - A thin wrapper around the [algoliasearch](https://ghub.io/algoliasearch) Node.js module for interacting with the Algolia API. +- [lib/algolia/search-index.js](lib/algolia/search-index.js) - A class for generating structured search data from repository content and syncing it with the remote Algolia service. This class has built-in validation to ensure that all records are valid before they're uploaded. This class also takes care of removing deprecated records, and compares existing remote records with the latest local records to avoid uploading records that haven't changed. +- [script/sync-algolia-search-indices.js](script/sync-algolia-search-indices.js) - The script used by the Actions workflow to update search indices on our Algolia account. This can also be [run in the development environment](#development). +- [tests/algolia-search.js](tests/algolia-search.js) - Tests! ## Indices From 0b3594cbb91441ee831f52fb71d3753f3aa7ea35 Mon Sep 17 00:00:00 2001 From: Sarah Schneider Date: Fri, 20 Nov 2020 16:58:02 -0500 Subject: [PATCH 09/10] a few more edits --- contributing/search.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/contributing/search.md b/contributing/search.md index 30919813b4c6..4da0507ebf9b 100644 --- a/contributing/search.md +++ b/contributing/search.md @@ -2,11 +2,11 @@ ## Table of contents - [Overview](#overview) -- [How it works](#how-it-works) +- [Production deploys](#production-deploys) - [Manual sync from a checkout](#manual-sync-from-a-checkout) - [Build without sync (dry run)](#build-without-sync-dry-run) - [Build and sync](#build-and-sync) -- [Label-triggered workflow](#label-triggered-workflow) +- [Label-triggered Actions workflow](#label-triggered-actions-workflow) - [Files](#files) - [GitHub Actions workflow files](#github-actions-workflow-files) - [Code files](#code-files) @@ -24,12 +24,14 @@ To see all existing search-related issues and pull requests, visit [github.com/g --- -## How it works +## Production deploys -The search data is synced automatically using a [GitHub Actions workflow](.github/workflows/sync-algolia-search-indices.yml) that is triggered by pushes to the `main` branch. This process generates structured data for all pages on the site, compares that data to what's currently on Algolia, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations. +A [GitHub Actions workflow](.github/workflows/sync-algolia-search-indices.yml) triggered by pushes to the `main` branch syncs the search data to Algolia. This process generates structured data for all pages on the site, compares that data to what's currently on Algolia, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations. The Actions workflow progress can be viewed (by GitHub employees) in the [Actions tab](https://github.com/github/docs/actions?query=workflow%3AAlgolia) of the repo. +Because the workflow runs after a branch is merged to `main`, there is a slight delay for search data updates to appear on the site. + ## Manual sync from a checkout It is also possible to manually sync the indices to Algolia from your local checkout of the repo, before your branch is merged to `main`. @@ -64,7 +66,7 @@ You can set `VERSION` and `LANGUAGE` individually, too. Substitute a currently supported version for `` and a currently supported two-letter language code for ``. -## Label-triggered workflow +## Label-triggered Actions workflow Docs team members can use an Actions workflow on GHES release PRs by applying a label in this format: ``` @@ -80,11 +82,11 @@ Why do we need this? For our daily shipping needs, it's tolerable that search up ## Files -### GitHub Actions workflow files +### Actions workflow files - [`.github/workflows/sync-algolia-search-indices.yml`](.github/workflows/sync-algolia-search-indices.yml) - Builds and syncs search indices whenever the `main` branch is pushed to (that is, on production deploys). - [`.github/workflows/dry-run-sync-algolia-search-indices.yml`](.github/workflows/dry-run-sync-algolia-search-indices.yml) - This workflow can be run manually (via `workflow_dispatch`) to do a dry run build of all the indices. Useful for confirming that the indices can build without erroring out. -- [`.github/workflows/sync-single-english-algolia-index.yml`](.github/workflows/sync-single-english-algolia-index.yml) - This workflow is run when a label in the right format is applied to a PR. See "[Label-triggered workflow](#label-triggered-workflow)" for details. +- [`.github/workflows/sync-single-english-algolia-index.yml`](.github/workflows/sync-single-english-algolia-index.yml) - This workflow is run when a label in the right format is applied to a PR. See "[Label-triggered Actions workflow](#label-triggered-actions-workflow)" for details. ### Code files From 18561c059f0a97711fd0b88dd4dc561bf83d98b5 Mon Sep 17 00:00:00 2001 From: Jason Etcovitch Date: Mon, 23 Nov 2020 15:06:57 -0500 Subject: [PATCH 10/10] Remove additional whitespace (#16593) --- lib/liquid-tags/extended-markdown.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/liquid-tags/extended-markdown.js b/lib/liquid-tags/extended-markdown.js index cf018b0845e1..1037b6d7c2ac 100644 --- a/lib/liquid-tags/extended-markdown.js +++ b/lib/liquid-tags/extended-markdown.js @@ -11,7 +11,7 @@ const tags = { danger: 'border rounded-1 mb-4 p-3 border-red bg-red-light f5' } -const template = '
\n{{ output }}\n
' +const template = '
{{ output }}
' class ExtendedMarkdown extends Liquid.Block { async render (context) {