fix: patch templates on the fly with seo robots meta tag for older versions

.github/workflows/go-release.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2

.github/workflows/go.yml

@@ -22,9 +22,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v6
         with:
           version: latest
           args: --timeout=30m
@@ -42,15 +42,13 @@ jobs:
     name: Test
     strategy:
       matrix:
-        go-version: [ 1.18.x ]
         platform: [ ubuntu-latest ]
     runs-on: ${{ matrix.platform }}
     steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with: { go-version-file: 'go.mod' }
       - name: Run tests with coverage
         run: go test -v -race ./...

README.md

@@ -52,6 +52,7 @@ The site data describes the location of its templates, assets, and content. It i
 - `redirects`: an object mapping URL paths (such as `/my/old/page`) to redirect destination URLs (such as `/my/new/page`).
 - `check` (optional): an object containing a single property `ignoreURLPattern`, which is a [RE2 regexp](https://golang.org/pkg/regexp/syntax/) of URLs to ignore when checking for broken URLs with `docsite check`.
 - `search` (optional): an object containing a single proprety `skipIndexURLPattern`, which is a [RE2 regexp](https://golang.org/pkg/regexp/syntax/) pattern that if matching any content file URL will remove that file from the search index.
+- `forceServedDownloadedContent` (optional) (dev):  While developing locally, you might want to see how docsite performs when it downloads the doc content remotely. With this set to true, docsite will download the content instead of serving from the filesystem
 
 The possible values for VFS URLs are:
 
@@ -115,6 +116,18 @@ The `docsite` tool requires site data to be available in any of the following wa
 
 ## Development
 
+## Running locally
+
+To run docsite locally and serve on port `:5080`, run:
+
+```shell
+go run ./cmd/docsite/... -config docsite.json serve
+```
+
+### Force serving downloaded content
+
+For certain use cases you want to have docsite download the docs content as it does with production configuration. To force this behaviour locally you can set `"forceServedDownloadedContent": true` in you `docsite.json` configuration
+
 ### Release a new version
 
 1. Build the Docker image for `linux/amd64`:

cmd/docsite/serve.go

@@ -3,10 +3,10 @@ package main
 import (
 	"crypto/tls"
 	"flag"
-	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
+	"os"
 	"sync"
 )
 
@@ -49,11 +49,11 @@ func init() {
 		}
 		if *tlsCertPath != "" || *tlsKeyPath != "" {
 			log.Printf("# TLS listener enabled")
-			tlsCert, err := ioutil.ReadFile(*tlsCertPath)
+			tlsCert, err := os.ReadFile(*tlsCertPath)
 			if err != nil {
 				return err
 			}
-			tlsKey, err := ioutil.ReadFile(*tlsKeyPath)
+			tlsKey, err := os.ReadFile(*tlsKeyPath)
 			if err != nil {
 				return err
 			}

cmd/docsite/site.go

@@ -6,7 +6,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"net/url"
@@ -36,7 +36,7 @@ func siteFromFlags() (*docsite.Site, *docsiteConfig, error) {
 
 	paths := filepath.SplitList(*configPath)
 	for _, path := range paths {
-		data, err := ioutil.ReadFile(path)
+		data, err := os.ReadFile(path)
 		if os.IsNotExist(err) {
 			continue
 		} else if err != nil {
@@ -52,16 +52,17 @@ func siteFromFlags() (*docsite.Site, *docsiteConfig, error) {
 // See ["Site data" in README.md](../../README.md#site-data) for documentation on this type's
 // fields.
 type docsiteConfig struct {
-	Content               string
-	ContentExcludePattern string
-	DefaultContentBranch  string
-	BaseURLPath           string
-	RootURL               string
-	Templates             string
-	Assets                string
-	AssetsBaseURLPath     string
-	Redirects             map[string]string
-	Check                 struct {
+	Content                     string
+	ContentExcludePattern       string
+	DefaultContentBranch        string
+	BaseURLPath                 string
+	RootURL                     string
+	Templates                   string
+	Assets                      string
+	AssetsBaseURLPath           string
+	ForceServeDownloadedContent bool
+	Redirects                   map[string]string
+	Check                       struct {
 		IgnoreURLPattern string
 	}
 	Search struct {
@@ -171,7 +172,6 @@ func addRedirectsFromAssets(site *docsite.Site) error {
 }
 
 const (
-	DEBUG        = false
 	CODEHOST_URL = "https://codeload.github.com/sourcegraph/sourcegraph-public-snapshot/zip/refs/heads/$VERSION#*/doc/"
 )
 
@@ -195,8 +195,10 @@ func openDocsiteFromConfig(configData []byte, baseDir string) (*docsite.Site, *d
 		return http.Dir(filepath.Join(baseDir, dir))
 	}
 
-	if DEBUG {
+	log.Printf("config %v", config)
+	if config.ForceServeDownloadedContent {
 		content := newVersionedFileSystemURL(CODEHOST_URL, "master")
+		log.Printf("Force serving content from %s", CODEHOST_URL)
 		if _, err := content.OpenVersion(context.Background(), ""); err != nil {
 			return nil, nil, errors.WithMessage(err, "downloading content default version")
 		}
@@ -322,7 +324,7 @@ func (fs *versionedFileSystemURL) fetchAndCacheVersion(version string) (http.Fil
 	if strings.Contains(urlStr, "$VERSION") && strings.Contains(urlStr, "github") && !strings.Contains(urlStr, "refs/heads/$VERSION") {
 		return nil, fmt.Errorf("refusing to use insecure docsite configuration for multi-version-aware GitHub URLs: the URL pattern %q must include \"refs/heads/$VERSION\", not just \"$VERSION\" (see docsite README.md for more information)", urlStr)
 	}
-	urlStr = strings.Replace(fs.url, "$VERSION", version, -1)
+	urlStr = strings.ReplaceAll(fs.url, "$VERSION", version)
 
 	// HACK: Workaround for https://github.com/sourcegraph/sourcegraph-public-snapshot/issues/3030. This assumes
 	// that tags all begin with "vN" where N is some number.
@@ -361,7 +363,7 @@ func zipFileSystemAtURL(url, dir string) (http.FileSystem, error) {
 	} else if resp.StatusCode != http.StatusOK {
 		return nil, &os.PathError{Op: "Get", Path: url, Err: fmt.Errorf("HTTP response status code %d", resp.StatusCode)}
 	}
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
 	}
@@ -395,7 +397,7 @@ func mapFromZipArchive(z *zip.Reader, dir string) (map[string]string, error) {
 		if err != nil {
 			return nil, errors.WithMessagef(err, "open %q", zf.Name)
 		}
-		data, err := ioutil.ReadAll(f)
+		data, err := io.ReadAll(f)
 		f.Close()
 		if err != nil {
 			return nil, errors.WithMessagef(err, "read %q", zf.Name)

config.json

@@ -0,0 +1 @@
+{"forceServeDownloadedContent": true, "templates":"https://codeload.github.com/sourcegraph/sourcegraph-public-snapshot/zip/legacydocs#*/doc/_resources/templates/","assets":"https://codeload.github.com/sourcegraph/sourcegraph-public-snapshot/zip/legacydocs#*/doc/_resources/assets/","content":"https://codeload.github.com/sourcegraph/sourcegraph-public-snapshot/zip/refs/heads/$VERSION#*/doc/","defaultContentBranch":"legacydocs","baseURLPath":"/","assetsBaseURLPath":"/assets/"}

go.mod

@@ -2,6 +2,7 @@ module github.com/sourcegraph/docsite
 
 require (
 	github.com/alecthomas/chroma v0.10.0
+	github.com/google/go-cmp v0.7.0
 	github.com/mozillazg/go-slugify v0.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/shurcooL/sanitized_anchor_name v1.0.0
@@ -21,4 +22,6 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-go 1.18
+go 1.21
+
+toolchain go1.23.2

go.sum

@@ -5,6 +5,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
 github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=

template.go

@@ -1,10 +1,11 @@
 package docsite
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"html/template"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
@@ -19,16 +20,29 @@ const (
 	rootTemplateName     = "root"
 	documentTemplateName = "document"
 	searchTemplateName   = "search"
+	metaRobots           = `<meta name="robots" content="noindex,nofollow">`
+	metaProperty         = `<meta property="og:locale" content="en_EN">`
 )
 
+func patchTemplateForSEO(data []byte) []byte {
+	if bytes.Contains(data, []byte(metaRobots)) {
+		return data
+	}
+
+	content := string(data)
+	content = strings.Replace(content, metaProperty, fmt.Sprintf("%s\n  %s", metaProperty, metaRobots), 1)
+
+	return []byte(content)
+}
+
 func (s *Site) getTemplate(templatesFS http.FileSystem, name string, extraFuncs template.FuncMap) (*template.Template, error) {
 	readFile := func(fs http.FileSystem, path string) ([]byte, error) {
 		f, err := fs.Open(path)
 		if err != nil {
 			return nil, err
 		}
 		defer f.Close()
-		data, err := ioutil.ReadAll(f)
+		data, err := io.ReadAll(f)
 		if err != nil {
 			return nil, err
 		}
@@ -99,6 +113,11 @@ func (s *Site) getTemplate(templatesFS http.FileSystem, name string, extraFuncs
 		if err != nil {
 			return nil, errors.WithMessage(err, fmt.Sprintf("read template %s", path))
 		}
+		if name == documentTemplateName {
+			// We need to patch the template, since if we're loading an old version the template won't have the
+			// nofollow, noindex seo tag
+			data = patchTemplateForSEO(data)
+		}
 		if _, err := tmpl.Parse(string(data)); err != nil {
 			return nil, errors.WithMessage(err, fmt.Sprintf("parse template %s", path))
 		}

template_test.go

@@ -0,0 +1,61 @@
+package docsite
+
+import (
+	"github.com/google/go-cmp/cmp"
+	"testing"
+)
+
+func TestPatchTempalteForSEO(t *testing.T) {
+	tt := []struct {
+		name    string
+		content string
+		want    string
+	}{
+		{
+			name: "content without robots meta is patched",
+			content: `
+{{define "seo"}}
+  <meta property="og:locale" content="en_EN">
+
+  <!-- Always set a title -->
+  {{ if .Content }}
+`,
+			want: `
+{{define "seo"}}
+  <meta property="og:locale" content="en_EN">
+  <meta name="robots" content="noindex,nofollow">
+
+  <!-- Always set a title -->
+  {{ if .Content }}
+`,
+		},
+		{
+			name: "content with robots meta is not patched",
+			content: `
+{{define "seo"}}
+  <meta property="og:locale" content="en_EN">
+  <meta name="robots" content="noindex,nofollow">
+
+  <!-- Always set a title -->
+  {{ if .Content }}
+`,
+			want: `
+{{define "seo"}}
+  <meta property="og:locale" content="en_EN">
+  <meta name="robots" content="noindex,nofollow">
+
+  <!-- Always set a title -->
+  {{ if .Content }}
+`,
+		},
+	}
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			got := string(patchTemplateForSEO([]byte(tc.content)))
+
+			if diff := cmp.Diff(tc.want, got); diff != "" {
+				t.Errorf("want and got mismatch (-want, +got): %s", diff)
+			}
+		})
+	}
+}

util.go

@@ -1,7 +1,7 @@
 package docsite
 
 import (
-	"io/ioutil"
+	"io"
 	"net/http"
 )
 
@@ -11,5 +11,5 @@ func ReadFile(fs http.FileSystem, path string) ([]byte, error) {
 		return nil, err
 	}
 	defer f.Close()
-	return ioutil.ReadAll(f)
+	return io.ReadAll(f)
 }