From 810b3db872b73cbc401653df549dd1197172c961 Mon Sep 17 00:00:00 2001 From: Lily Wang Date: Sun, 6 Dec 2020 15:26:39 +1100 Subject: [PATCH 1/4] add gh-ci --- .github/workflows/gh-ci.yaml | 116 +++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 .github/workflows/gh-ci.yaml diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml new file mode 100644 index 000000000..32dc5427a --- /dev/null +++ b/.github/workflows/gh-ci.yaml @@ -0,0 +1,116 @@ +name: mda_gh_ci +on: + push: + branches: + - develop + - master + pull_request: + branches: + - develop + - master + +defaults: + run: + shell: bash -l {0} + +env: + CONDA_DEPS: "jupyter_contrib_nbextensions nglview sphinx ipywidgets matplotlib==3.2.2 MDAnalysis MDAnalysisTests plotly" + PIP_DEPS: "sphinx_rtd_theme sphinx-sitemap nbsphinx ipython tabulate sphinxcontrib-bibtex pybtex msmb_theme==1.2.0" + SPHINX_DIR: "doc/" + HTML_DIR: "doc/build/html" + + +jobs: + build_docs: + if: "github.repository == 'MDAnalysis/UserGuide'" + runs-on: ubuntu-latest + env: + CYTHON_TRACE_NOGIL: 1 + MPLBACKEND: agg + + steps: + - uses: actions/checkout@v2 + + - name: setup_miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.7 + auto-update-conda: true + channel-priority: flexible + channels: plotly, conda-forge + add-pip-as-python-dependency: true + architecture: x64 + + - name: install_deps + run: | + conda install ${{ env.CONDA_DEPS }} + pip install ${{ env.PIP_DEPS }} + jupyter-nbextension enable nglview --py --sys-prefix + + - name: build_docs + run: | + make -C ${SPHINX_DIR} html + + - name: deploy_docs + if: github.event_name != 'pull_request' + env: + GH_USER: github-actions + GH_EMAIL: "github-action@users.noreply.github.com" + GH_REPOSITORY: "github.com/${{ github.repository }}.git" + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + URL: https://userguide.mdanalysis.org + + run: | + # set up environment variables + # cannot execute bash to make variables in env section + # export URL for the Python script $UPDATE_JSON + export URL + export VERSION=$(python -c "import MDAnalysis; print(MDAnalysis.__version__)") + UPDATE_JSON=$(pwd)/maintainer/update_json_stubs_sitemap.py + BRANCH="${GITHUB_REF#refs/heads/}" + + # the below turns off non-blocking as it causes large writes to stdout to fail + # (see https://github.com/travis-ci/travis-ci/issues/4704) + # commented out as this is not a problem with gh-actions + # python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' + cd ${{ env.HTML_DIR }} + + # move docs into version subfolder + mkdir ../${VERSION} && mv * ../${VERSION} && mv ../${VERSION} $VERSION + + # set up git + REV=$(git rev-parse --short HEAD) + git init + git config user.name $GH_USER + git config user.email $GH_EMAIL + git remote add upstream "https://${GH_USER}:${GH_TOKEN}@${GH_REPOSITORY}" + git fetch --depth 50 upstream $BRANCH gh-pages + git reset upstream/gh-pages + + # redirects and copies + mkdir latest + python $UPDATE_JSON + touch . + touch .nojekyll + + git add -A ${VERSION}/ + git add .nojekyll versions.json *.xml *.html index.html latest + + for dirname in dev stable ; do + if [ -d $dirname ]; then git add $dirname; fi + done + + # add redirect html files if they're generated + if [ "$(ls *.html | wc -l)" -ge "1" ]; then + # get list of directories and files present in the latest version + cd $VERSION && mdfiles=$(ls -d *) && cd - + for item in $mdfiles ; do + # if the directory/file exists in top level, add it + if [[ -f $item ]] || [[ -d $item ]]; then git add $item; fi + done + fi + + # check for anything to commit + # https://stackoverflow.com/questions/3878624/how-do-i-programmatically-determine-if-there-are-uncommited-changes + git diff-index --quiet HEAD -- || git commit -m "rebuilt html docs for version ${VERSION} from branch ${BRANCH} with sphinx at ${REV}" + git push -q upstream HEAD:gh-pages From 17f7b0fe9e78ea4c5cb50ee55f676377f2c2259d Mon Sep 17 00:00:00 2001 From: Lily Wang Date: Sun, 6 Dec 2020 15:27:06 +1100 Subject: [PATCH 2/4] rename travis --- .travis.yml => .disabled-travis.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .travis.yml => .disabled-travis.yml (100%) diff --git a/.travis.yml b/.disabled-travis.yml similarity index 100% rename from .travis.yml rename to .disabled-travis.yml From b6eff7ea133e9d703b873a9bed0bf6b268e53a3e Mon Sep 17 00:00:00 2001 From: Lily Wang Date: Sun, 6 Dec 2020 15:29:47 +1100 Subject: [PATCH 3/4] to undo: change site to lilyminium's --- .github/workflows/gh-ci.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml index 32dc5427a..8560a0643 100644 --- a/.github/workflows/gh-ci.yaml +++ b/.github/workflows/gh-ci.yaml @@ -22,7 +22,6 @@ env: jobs: build_docs: - if: "github.repository == 'MDAnalysis/UserGuide'" runs-on: ubuntu-latest env: CYTHON_TRACE_NOGIL: 1 @@ -52,13 +51,12 @@ jobs: make -C ${SPHINX_DIR} html - name: deploy_docs - if: github.event_name != 'pull_request' env: GH_USER: github-actions GH_EMAIL: "github-action@users.noreply.github.com" - GH_REPOSITORY: "github.com/${{ github.repository }}.git" + GH_REPOSITORY: "github.com/lilyminium/UserGuide.git" GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - URL: https://userguide.mdanalysis.org + URL: http://minium.com.au/UserGuide run: | # set up environment variables From 2b11c6a85e7baeb705d261fccd04026960ca8924 Mon Sep 17 00:00:00 2001 From: Lily Wang Date: Sun, 6 Dec 2020 15:41:06 +1100 Subject: [PATCH 4/4] add stable and dev versions --- maintainer/update_json_stubs_sitemap.py | 109 +++++++++++++++++++----- 1 file changed, 90 insertions(+), 19 deletions(-) diff --git a/maintainer/update_json_stubs_sitemap.py b/maintainer/update_json_stubs_sitemap.py index 32c9fcbd2..f6b1436ab 100644 --- a/maintainer/update_json_stubs_sitemap.py +++ b/maintainer/update_json_stubs_sitemap.py @@ -11,11 +11,12 @@ import json import os +import shutil +import xml.etree.ElementTree as ET import errno import glob -import shutil import textwrap -import xml.etree.ElementTree as ET +import shutil try: from urllib.request import Request, urlopen @@ -25,6 +26,16 @@ URL = os.environ['URL'] VERSION = os.environ['VERSION'] +if "http" not in URL: + raise ValueError("URL should have the transfer protocol (HTTP/S). " + f"Given: $URL={URL}") + +try: + int(VERSION[0]) +except ValueError: + raise ValueError("$VERSION should start with a number. " + f"Given: $VERSION={VERSION}") from None + def get_web_file(filename, callback, default): url = os.path.join(URL, filename) @@ -43,13 +54,6 @@ def get_web_file(filename, callback, default): return callback(data) -# ========= WRITE HTML STUBS ========= -# Add HTML files to redirect: -# index.html -> latest release -# latest/index.html -> latest release -# dev/index.html -> dev docs - - def write_redirect(file, version='', outfile=None): if outfile is None: outfile = file @@ -63,6 +67,7 @@ def write_redirect(file, version='', outfile=None): """) with open(outfile, 'w') as f: f.write(REDIRECT) + print(f"Wrote redirect from {url} to {outfile}") # ========= WRITE JSON ========= @@ -70,9 +75,9 @@ def write_redirect(file, version='', outfile=None): versions = get_web_file('versions.json', json.loads, []) existing = [item['version'] for item in versions] already_exists = VERSION in existing +latest = 'dev' not in VERSION if not already_exists: - latest = 'dev' not in VERSION if latest: for ver in versions: ver['latest'] = False @@ -84,9 +89,6 @@ def write_redirect(file, version='', outfile=None): 'latest': latest }) -with open("versions.json", 'w') as f: - json.dump(versions, f, indent=2) - for ver in versions[::-1]: if ver['latest']: latest_version = ver['version'] @@ -98,7 +100,7 @@ def write_redirect(file, version='', outfile=None): latest_version = None for ver in versions[::-1]: - if 'dev' in ver['version']: + if '-dev' in ver['version']: dev_version = ver['version'] break else: @@ -107,10 +109,68 @@ def write_redirect(file, version='', outfile=None): except IndexError: dev_version = None -if latest_version: - html_files = glob.glob(f'{latest_version}/**/*.html', recursive=True) +versions.sort(key=lambda x: x["version"]) + +# ========= WRITE HTML STUBS AND COPY DOCS ========= +# Add HTML files to redirect: +# index.html -> stable/ docs +# latest/index.html -> latest release (not dev docs) +# stable/ : a copy of the release docs with the highest number (2.0.0 instead of 1.0.0) +# dev/ : a copy of the develop docs with the highest number (2.0.0-dev instead of 1.0.1-dev) +# sitemap.xml files are updated by replacing URL strings + + +def redirect_sitemap(old_version, new_version): + """Replace paths in copied sitemap.xml with new directory path + + Sitemaps can only contain URLs 'within' that directory structure. + For more, see https://www.sitemaps.org/faq.html#faq_sitemap_location + """ + file = f"{new_version}/sitemap.xml" + old = f"{URL}/{old_version}/" + new = f"{URL}/{new_version}/" + try: + with open(file, "r") as f: + contents = f.read() + except OSError: + raise ValueError(f"{file} not found") + redirected = contents.replace(old, new) + with open(file, "w") as f: + f.write(redirected) + print(f"Redirected URLs in {file} from {old} to {new}") + + +def add_or_update_version(version): + """Add or update the version path to versions.json""" + for ver in versions: + if ver["version"] == version: + ver["url"] = os.path.join(URL, version) + break + else: + versions.append({ + "version": version, + "display": version, + "url": os.path.join(URL, version), + "latest": False + }) + + +def copy_version(old_version, new_version): + """Copy docs from one directory to another with all bells and whistles""" + shutil.copytree(old_version, new_version) + print(f"Copied {old_version} to {new_version}") + redirect_sitemap(old_version, new_version) + add_or_update_version(new_version) + + +# Copy stable/ docs and write redirects from root level docs +if latest: + copy_version(VERSION, "stable") + html_files = glob.glob(f'stable/**/*.html', recursive=True) for file in html_files: - outfile = file.strip(f'{latest_version}/') + # below should be true because we only globbed stable/* paths + assert file.startswith("stable/") + outfile = file[7:] # strip "stable/" dirname = os.path.dirname(outfile) if dirname and not os.path.exists(dirname): try: @@ -120,10 +180,21 @@ def write_redirect(file, version='', outfile=None): raise write_redirect(file, '', outfile) + +# Separate just in case we update versions.json or muck around manually +# with docs +if latest_version: + write_redirect('index.html', "stable") write_redirect('index.html', latest_version, 'latest/index.html') -if dev_version: - write_redirect('index.html', dev_version, 'dev/index.html') +# Copy dev/ docs +if dev_version and dev_version == VERSION: + copy_version(VERSION, "dev") + +# update versions.json online +with open("versions.json", 'w') as f: + json.dump(versions, f, indent=2) + # ========= WRITE SUPER SITEMAP.XML ========= # make one big sitemap.xml