Add GH actions docs deployment and stable/dev versioning

.github/workflows/gh-ci.yaml

@@ -0,0 +1,114 @@
+name: mda_gh_ci
+on:
+  push:
+    branches:
+      - develop
+      - master
+  pull_request:
+    branches:
+      - develop
+      - master
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+env:
+  CONDA_DEPS: "jupyter_contrib_nbextensions nglview sphinx ipywidgets matplotlib==3.2.2 MDAnalysis MDAnalysisTests plotly"
+  PIP_DEPS: "sphinx_rtd_theme sphinx-sitemap nbsphinx ipython tabulate sphinxcontrib-bibtex pybtex msmb_theme==1.2.0"
+  SPHINX_DIR: "doc/"
+  HTML_DIR: "doc/build/html"
+
+
+jobs:
+  build_docs:
+    runs-on: ubuntu-latest
+    env:
+      CYTHON_TRACE_NOGIL: 1
+      MPLBACKEND: agg
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: setup_miniconda
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        python-version: 3.7
+        auto-update-conda: true
+        channel-priority: flexible
+        channels: plotly, conda-forge
+        add-pip-as-python-dependency: true
+        architecture: x64
+
+    - name: install_deps
+      run: |
+        conda install ${{ env.CONDA_DEPS }}
+        pip install ${{ env.PIP_DEPS }}
+        jupyter-nbextension enable nglview --py --sys-prefix
+
+    - name: build_docs
+      run: |
+        make -C ${SPHINX_DIR} html
+
+    - name: deploy_docs
+      env:
+        GH_USER: github-actions
+        GH_EMAIL: "github-action@users.noreply.github.com"
+        GH_REPOSITORY: "github.com/lilyminium/UserGuide.git"
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        URL: http://minium.com.au/UserGuide
+
+      run: |
+        # set up environment variables
+        # cannot execute bash to make variables in env section
+        # export URL for the Python script $UPDATE_JSON
+        export URL
+        export VERSION=$(python -c "import MDAnalysis; print(MDAnalysis.__version__)")
+        UPDATE_JSON=$(pwd)/maintainer/update_json_stubs_sitemap.py
+        BRANCH="${GITHUB_REF#refs/heads/}"
+
+        # the below turns off non-blocking as it causes large writes to stdout to fail
+        # (see https://github.com/travis-ci/travis-ci/issues/4704)
+        # commented out as this is not a problem with gh-actions
+        # python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
+        cd ${{ env.HTML_DIR }}
+
+        # move docs into version subfolder
+        mkdir ../${VERSION} && mv * ../${VERSION} && mv ../${VERSION} $VERSION
+
+        # set up git
+        REV=$(git rev-parse --short HEAD)
+        git init
+        git config user.name $GH_USER
+        git config user.email $GH_EMAIL
+        git remote add upstream "https://${GH_USER}:${GH_TOKEN}@${GH_REPOSITORY}"
+        git fetch --depth 50 upstream $BRANCH gh-pages
+        git reset upstream/gh-pages
+
+        # redirects and copies
+        mkdir latest
+        python $UPDATE_JSON
+        touch .
+        touch .nojekyll
+
+        git add -A ${VERSION}/
+        git add .nojekyll versions.json *.xml *.html index.html latest
+
+        for dirname in dev stable ; do
+          if [ -d $dirname ]; then git add $dirname; fi
+        done
+
+        # add redirect html files if they're generated
+        if [ "$(ls *.html | wc -l)" -ge "1" ]; then
+            # get list of directories and files present in the latest version
+            cd $VERSION && mdfiles=$(ls -d *) && cd -
+            for item in $mdfiles ; do
+                # if the directory/file exists in top level, add it
+                if [[ -f $item ]] || [[ -d $item ]]; then git add $item; fi
+            done
+        fi
+
+        # check for anything to commit
+        # https://stackoverflow.com/questions/3878624/how-do-i-programmatically-determine-if-there-are-uncommited-changes
+        git diff-index --quiet HEAD -- || git commit -m "rebuilt html docs for version ${VERSION} from branch ${BRANCH} with sphinx at ${REV}"
+        git push -q upstream HEAD:gh-pages

maintainer/update_json_stubs_sitemap.py

@@ -11,11 +11,12 @@
 
 import json
 import os
+import shutil
+import xml.etree.ElementTree as ET
 import errno
 import glob
-import shutil
 import textwrap
-import xml.etree.ElementTree as ET
+import shutil
 
 try:
     from urllib.request import Request, urlopen
@@ -25,6 +26,16 @@
 URL = os.environ['URL']
 VERSION = os.environ['VERSION']
 
+if "http" not in URL:
+    raise ValueError("URL should have the transfer protocol (HTTP/S). "
+                     f"Given: $URL={URL}")
+
+try:
+    int(VERSION[0])
+except ValueError:
+    raise ValueError("$VERSION should start with a number. "
+                     f"Given: $VERSION={VERSION}") from None
+
 
 def get_web_file(filename, callback, default):
     url = os.path.join(URL, filename)
@@ -43,13 +54,6 @@ def get_web_file(filename, callback, default):
         return callback(data)
 
 
-# ========= WRITE HTML STUBS =========
-# Add HTML files to redirect:
-# index.html -> latest release
-# latest/index.html -> latest release
-# dev/index.html -> dev docs
-
-
 def write_redirect(file, version='', outfile=None):
     if outfile is None:
         outfile = file
@@ -63,16 +67,17 @@ def write_redirect(file, version='', outfile=None):
     """)
     with open(outfile, 'w') as f:
         f.write(REDIRECT)
+    print(f"Wrote redirect from {url} to {outfile}")
 
 
 # ========= WRITE JSON =========
 # Update $root/versions.json with links to the right version
 versions = get_web_file('versions.json', json.loads, [])
 existing = [item['version'] for item in versions]
 already_exists = VERSION in existing
+latest = 'dev' not in VERSION
 
 if not already_exists:
-    latest = 'dev' not in VERSION
     if latest:
         for ver in versions:
             ver['latest'] = False
@@ -84,9 +89,6 @@ def write_redirect(file, version='', outfile=None):
         'latest': latest
     })
 
-with open("versions.json", 'w') as f:
-    json.dump(versions, f, indent=2)
-
 for ver in versions[::-1]:
     if ver['latest']:
         latest_version = ver['version']
@@ -98,7 +100,7 @@ def write_redirect(file, version='', outfile=None):
         latest_version = None
 
 for ver in versions[::-1]:
-    if 'dev' in ver['version']:
+    if '-dev' in ver['version']:
         dev_version = ver['version']
         break
 else:
@@ -107,10 +109,68 @@ def write_redirect(file, version='', outfile=None):
     except IndexError:
         dev_version = None
 
-if latest_version:
-    html_files = glob.glob(f'{latest_version}/**/*.html', recursive=True)
+versions.sort(key=lambda x: x["version"])
+
+# ========= WRITE HTML STUBS AND COPY DOCS =========
+# Add HTML files to redirect:
+# index.html -> stable/ docs
+# latest/index.html -> latest release (not dev docs)
+# stable/ : a copy of the release docs with the highest number (2.0.0 instead of 1.0.0)
+# dev/ : a copy of the develop docs with the highest number (2.0.0-dev instead of 1.0.1-dev)
+# sitemap.xml files are updated by replacing URL strings
+
+
+def redirect_sitemap(old_version, new_version):
+    """Replace paths in copied sitemap.xml with new directory path
+
+    Sitemaps can only contain URLs 'within' that directory structure.
+    For more, see https://www.sitemaps.org/faq.html#faq_sitemap_location
+    """
+    file = f"{new_version}/sitemap.xml"
+    old = f"{URL}/{old_version}/"
+    new = f"{URL}/{new_version}/"
+    try:
+        with open(file, "r") as f:
+            contents = f.read()
+    except OSError:
+        raise ValueError(f"{file} not found")
+    redirected = contents.replace(old, new)
+    with open(file, "w") as f:
+        f.write(redirected)
+    print(f"Redirected URLs in {file} from {old} to {new}")
+
+
+def add_or_update_version(version):
+    """Add or update the version path to versions.json"""
+    for ver in versions:
+        if ver["version"] == version:
+            ver["url"] = os.path.join(URL, version)
+            break
+    else:
+        versions.append({
+            "version": version,
+            "display": version,
+            "url": os.path.join(URL, version),
+            "latest": False
+        })
+
+
+def copy_version(old_version, new_version):
+    """Copy docs from one directory to another with all bells and whistles"""
+    shutil.copytree(old_version, new_version)
+    print(f"Copied {old_version} to {new_version}")
+    redirect_sitemap(old_version, new_version)
+    add_or_update_version(new_version)
+
+
+# Copy stable/ docs and write redirects from root level docs
+if latest:
+    copy_version(VERSION, "stable")
+    html_files = glob.glob(f'stable/**/*.html', recursive=True)
     for file in html_files:
-        outfile = file.strip(f'{latest_version}/')
+        # below should be true because we only globbed stable/* paths
+        assert file.startswith("stable/")
+        outfile = file[7:]  # strip "stable/"
         dirname = os.path.dirname(outfile)
         if dirname and not os.path.exists(dirname):
             try:
@@ -120,10 +180,21 @@ def write_redirect(file, version='', outfile=None):
                     raise
 
         write_redirect(file, '', outfile)
+
+# Separate just in case we update versions.json or muck around manually
+# with docs
+if latest_version:
+    write_redirect('index.html', "stable")
     write_redirect('index.html', latest_version, 'latest/index.html')
 
-if dev_version:
-    write_redirect('index.html', dev_version, 'dev/index.html')
+# Copy dev/ docs
+if dev_version and dev_version == VERSION:
+    copy_version(VERSION, "dev")
+
+# update versions.json online
+with open("versions.json", 'w') as f:
+    json.dump(versions, f, indent=2)
+
 
 # ========= WRITE SUPER SITEMAP.XML =========
 # make one big sitemap.xml