From 9742cfa4c46de6c3740896db86df6a86236b58b6 Mon Sep 17 00:00:00 2001 From: Irfan Alibay Date: Wed, 2 Dec 2020 18:56:39 +0000 Subject: [PATCH 1/7] Adds basic GH Actions CI workflow (#3040) Fixes #3036 ## Work done in this PR - Adds a continuous integration workflow based on github actions. - Disables previous continuous integration workflow based on TravisCI. - Fixes minor Visibledeprecation warnings with the hole2 tests. --- .travis.yml => .disabled-travis.yml | 0 .github/workflows/gh-ci.yaml | 281 ++++++++++++++++++ maintainer/install_hole.sh | 115 ------- .../MDAnalysisTests/analysis/test_hole2.py | 6 +- 4 files changed, 284 insertions(+), 118 deletions(-) rename .travis.yml => .disabled-travis.yml (100%) create mode 100644 .github/workflows/gh-ci.yaml delete mode 100755 maintainer/install_hole.sh diff --git a/.travis.yml b/.disabled-travis.yml similarity index 100% rename from .travis.yml rename to .disabled-travis.yml diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml new file mode 100644 index 00000000000..a9fee810031 --- /dev/null +++ b/.github/workflows/gh-ci.yaml @@ -0,0 +1,281 @@ +name: mda_gh_ci +on: + push: + branches: + - develop + - master + pull_request: + branches: + - develop + - master + +defaults: + run: + shell: bash -l {0} + +env: + MDA_CONDA_MIN_DEPS: "pip pytest mmtf-python biopython networkx cython matplotlib scipy griddataformats hypothesis gsd codecov" + MDA_CONDA_EXTRA_DEPS: "seaborn>=0.7.0 clustalw=2.1 netcdf4 scikit-learn joblib>=0.12 chemfiles tqdm>=4.43.0 tidynamics>=1.0.0 rdkit>=2020.03.1 h5py==2.10.0" + MDA_PIP_MIN_DEPS: 'coveralls coverage<5 pytest-cov pytest-xdist' + MDA_PIP_EXTRA_DEPS: 'duecredit parmed' + +jobs: + main_tests: + if: "github.repository == 'MDAnalysis/mdanalysis'" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ] + python-version: [3.6, 3.7, 3.8] + run_type: [FULL, ] + install_hole: [true, ] + codecov: [true, ] + include: + - name: macOS + os: macOS-latest + python-version: 3.7 + run_type: FULL + install_hole: true + codecov: true + - name: minimal-ubuntu + os: ubuntu-latest + python-version: 3.6 + run_type: MIN + install_hole: false + codecov: true + - name: numpy_min + os: ubuntu-latest + python-version: 3.6 + run_type: FULL + install_hole: false + codecov: false + numpy: numpy=1.16.0 + - name: asv_check + os: ubuntu-latest + python-version: 3.7 + run_type: FULL + install_hole: false + codecov: false + env: + CYTHON_TRACE_NOGIL: 1 + MPLBACKEND: agg + GH_OS: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + + - name: setup_osx + if: startsWith(matrix.os, 'macOS') + run: | + # Set OS specific vars and compiler flags + echo "OS_NAME=osx" >> $GITHUB_ENV + # TODO: work out why this is necessary (from CI helpers) + echo "MACOSX_DEPLOYMENT_TARGET=10.9" >> $GITHUB_ENV + ulimit -S -n 2048 + clang -v + echo "CC=clang" >> $GITHUB_ENV + clang++ -v + echo "CXX=clang++" >> $GITHUB_ENV + gfortran-9 -v + echo "FC=gfortran-9" >> $GITHUB_ENV + + - name: setup_linux + if: startsWith(matrix.os, 'ubuntu') + run: | + # Set OS specific vars and compiler flags + echo "OS_NAME=linux" >> $GITHUB_ENV + gcc -v + echo "CC=gcc" >> $GITHUB_ENV + g++ -v + echo "CXX=g++" >> $GITHUB_ENV + gfortran -v + echo "FC=gfortran" >> $GITHUB_ENV + + - name: setup_miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: ${{ matrix.python-version }} + auto-update-conda: true + channel-priority: flexible + channels: biobuilds, conda-forge + add-pip-as-python-dependency: true + # TODO: mamba causes pip to segfault, switch when fixed + #mamba-version: "*" + architecture: x64 + + - name: install_deps + env: + MDA_CONDA_FULL_DEPS: "${{ env.MDA_CONDA_MIN_DEPS }} ${{ env.MDA_CONDA_EXTRA_DEPS }}" + MDA_PIP_FULL_DEPS: "${{ env.MDA_PIP_MIN_DEPS }} ${{ env.MDA_PIP_EXTRA_DEPS }}" + run: | + # NOTE: vars need to be re-assigned + # NOTE: set matrix.numpy to pin to a specific numpy version + conda_deps="${{ matrix.numpy }} ${MDA_CONDA_${{ matrix.run_type }}_DEPS}" + pip_deps=${MDA_PIP_${{ matrix.run_type }}_DEPS} + conda install ${conda_deps} + pip install ${pip_deps} + + # also install asv if required + if [ ${{ matrix.name }} = "asv_check" ]; then + pip install asv + fi + + - name: check_setup + run: | + # Check OS and python setup + echo "OS: ${OS_NAME}" + which python + which pip + pip list + conda info + conda list + + - name: install_hole + if : matrix.install_hole + run: | + # We manually build hole2 to avoid OS incompatibilities + git clone https://github.com/MDAnalysis/hole2.git + cd hole2/src + source ../source.apache + (make FC=${FC}) && (make PREFIX=${HOME}/hole2 FC=${FC} install) + source ../source.unset + echo "HOLE_BINDIR=${HOME}/hole2/bin" >> $GITHUB_ENV + echo "${HOME}/hole2/bin" >> $GITHUB_PATH + + - name: install_mda + run: | + # TODO: using install instead of develop here causes coverage to drop + # for .pyx file. If possible a solution for this should be found. + (cd package/ && python setup.py develop) && (cd testsuite/ && python setup.py install) + + - name: run_tests + if: contains(matrix.name, 'asv_check') != true + run: | + PYTEST_FLAGS="--disable-pytest-warnings --durations=50" + if [ ${{ matrix.codecov }} = "true" ]; then + PYTEST_FLAGS="${PYTEST_FLAGS} --cov=MDAnalysis --cov-report=xml" + fi + echo $PYTEST_FLAGS + pytest -n 2 testsuite/MDAnalysisTests $PYTEST_FLAGS + + - name: run_asv + if: contains(matrix.name, 'asv_check') + run: | + cd benchmarks + time python -m asv check -E existing + + - name: codecov + if: matrix.codecov + uses: codecov/codecov-action@v1 + with: + file: coverage.xml + fail_ci_if_error: True + verbose: True + + + build_docs: + if: "github.repository == 'MDAnalysis/mdanalysis'" + runs-on: ubuntu-latest + env: + CYTHON_TRACE_NOGIL: 1 + MPLBACKEND: agg + + steps: + - uses: actions/checkout@v2 + + - name: setup_miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.7 + auto-update-conda: true + channel-priority: flexible + channels: biobuilds, conda-forge + add-pip-as-python-dependency: true + architecture: x64 + + - name: install_deps + run: | + conda_deps="${{ env.MDA_CONDA_MIN_DEPS }} ${{ env.MDA_CONDA_EXTRA_DEPS}}" + pip_deps="${{ env.MDA_PIP_MIN_DEPS}} ${{ env.MDA_PIP_EXTRA_DEPS }} sphinx==1.8.5 sphinx-sitemap sphinx_rtd_theme msmb_theme==1.2.0" + conda install ${conda_deps} + pip install ${pip_deps} + + - name: install_mda + run: | + cd package && python setup.py develop + + - name: build_docs + run: | + cd package && python setup.py build_sphinx -E + + - name: deploy_docs + if: github.event_name != 'pull_request' + run: | + # place the deploy call here + echo "Oh, maple syrup roast parsnips [Richard Gowers]" + + + pylint_check: + if: "github.repository == 'MDAnalysis/mdanalysis'" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: setup_miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.7 + auto-update-conda: true + channel-priority: flexible + channels: conda-forge + add-pip-as-python-dependency: true + mamba-version: "*" + architecture: x64 + + - name: install + run: | + which pip + which python + pip install pylint + + - name: pylint + env: + PYLINTRC: package/.pylintrc + run: | + pylint --py3k package/MDAnalysis && pylint --py3k testsuite/MDAnalysisTests + + + pypi_check: + if: "github.repository == 'MDAnalysis/mdanalysis'" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: setup_miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.7 + auto-update-conda: true + channel-priority: flexible + channels: conda-forge + add-pip-as-python-dependency: true + mamba-version: "*" + architecture: x64 + + - name: install_conda + run: | + conda install setuptools cython numpy twine + + - name: install_mdanalysis + run: | + cd package && python setup.py sdist + + - name: check_package_build + run: | + DISTRIBUTION=$(ls -t1 package/dist/MDAnalysis-*.tar.gz | head -n 1) + test -n "${DISTRIBUTION}" || { echo "no distribution package/dist/MDAnalysis-*.tar.gz found"; exit 1; } + echo "twine check $DISTRIBUTION" + twine check $DISTRIBUTION diff --git a/maintainer/install_hole.sh b/maintainer/install_hole.sh deleted file mode 100755 index 8e2046ae310..00000000000 --- a/maintainer/install_hole.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# -# Written by Oliver Beckstein, 2016 -# -# This script is placed into the Public Domain using the CC0 1.0 Public Domain -# Dedication https://creativecommons.org/publicdomain/zero/1.0/ -# -# -# NOTE: IF YOU RUN THIS SCRIPT YOU MUST BE ABLE TO COMPLY WITH THE HOLE -# NOT-FOR-PROFIT LICENSE. -# -# -# Install HOLE from http://www.holeprogram.org/ -# -# arguments -# OSNAME : linux | darwin -# PREFIX : "path/to/installdir" -# -# PREFIX can be relative to CWD or an absolute path; it will be created -# and HOLE unpacked under PREFIX (the tar file contains "hole2/...") -# -# -# HOLE v2.2004 is used under the terms of the 'HOLE END USER LICENCE AGREEMENT -# NOT-FOR-PROFIT VERSION' as provided in the installation tarball as file -# 'doc/Licence-not-for-profit.asciidoc' and see copy at -# https://github.com/MDAnalysis/mdanalysis/files/372246/Licence-not-for-profit.txt -# (recent as of 2016-07-19). -# - - -set -o errexit -o nounset - -SCRIPT=$(basename $0) - -# We still use the 2.2004 versions (academic only) because -# for the v2.2005 (Apache license) there are no pre-compiled binaries -# available at http://www.holeprogram.org/ as of 2018-02-22 - -# could switch to http://www.holeprogram.org/downloads/2.2.004/hole2-NotForProfit-2.2.004-Linux-i686.tar.gz -DOWNLOAD_URL_LINUX='https://www.holeprogram.org/downloads/2.2.004/hole2-NotForProfit-2.2.004-Linux-x86_64.tar.gz' -TARFILE_LINUX='hole2-NotForProfit-2.2.004-Linux-x86_64.tar.gz' - -# could switch to http://www.holeprogram.org/downloads/2.2.004/hole2-NotForProfit-2.2.004-Darwin-i386.tar.gz -DOWNLOAD_URL_DARWIN='https://www.holeprogram.org/downloads/2.2.004/hole2-NotForProfit-2.2.004-Darwin-i386.tar.gz' -TARFILE_DARWIN=hole2-NotForProfit-2.2.004-Darwin-i386.tar.gz - -# path to dir with executables in the current HOLE distribution -HOLE_EXE_DIR=hole2/exe - -function die () { - local msg="$1" err=$2 - echo "[${SCRIPT}] ERROR: $msg [$err]" - exit $err -} - -function is_native_executable () { - local filename="$1" OSNAME="$2" - file "${filename}" | grep -qi ${OFORMAT} - return $? -} - -OSNAME="$1" -case "$OSNAME" in - Linux|linux) - OSNAME=Linux - OFORMAT=Linux - DOWNLOAD_URL="${DOWNLOAD_URL_LINUX}" - TARFILE=${TARFILE_LINUX} - ;; - OSX|osx|Darwin|darwin) - OSNAME=Darwin - OFORMAT="Mach-O" - DOWNLOAD_URL="${DOWNLOAD_URL_DARWIN}" - TARFILE=${TARFILE_DARWIN} - ;; - *) - die "OS '${OSNAME}' not supported." 10;; -esac - -PREFIX="$2" -test -n "$PREFIX" || die "missing second argument PREFIX (installation directory)" 10 - -#------------------------------------------------------------ -# start installation -#------------------------------------------------------------ - -mkdir -p "$PREFIX" && cd "$PREFIX" || die "Failed to create and cd to $PREFIX" 1 -if ! test -f ${TARFILE}; then - echo "Downloading ${TARFILE} from ${DOWNLOAD_URL}..." - # fixing curl on travis/anaconda, see http://stackoverflow.com/a/31060428/334357 - export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt - curl -L ${DOWNLOAD_URL} -o ${TARFILE} || \ - die "Failed to download ${TARFILE} from ${DOWNLOAD_URL}" 1 -fi - -# only install the executables in HOLE_EXE_DIR -tar xvf ${TARFILE} ${HOLE_EXE_DIR} || die "Failed 'tar xvf ${TARFILE} ${HOLE_EXE_DIR}'" 1 - -MDA_HOLE_BINDIR="${PWD}/${HOLE_EXE_DIR}" -HOLE_EXE="${MDA_HOLE_BINDIR}/hole" - -test -d "${MDA_HOLE_BINDIR}" || { \ - echo "Content of ${PWD}:"; - ls -la; - die "no HOLE exe dir ${MDA_HOLE_BINDIR} in $PWD" 2; } -test -f "${HOLE_EXE}" || die "hole executable ${HOLE_EXE} not installed" 2 -is_native_executable ${HOLE_EXE} ${OFORMAT} || \ - { file ${HOLE_EXE}; \ - die "${HOLE_EXE} will not run on ${OSNAME} (object format ${OFORMAT})" 3; } - -echo "HOLE executables were installed into ${MDA_HOLE_BINDIR}" -echo ">>> export PATH=\${PATH}:${MDA_HOLE_BINDIR}" - -# repeat this line in .travis.yml -export PATH=${PATH}:${MDA_HOLE_BINDIR} diff --git a/testsuite/MDAnalysisTests/analysis/test_hole2.py b/testsuite/MDAnalysisTests/analysis/test_hole2.py index 700827f027f..580b28485aa 100644 --- a/testsuite/MDAnalysisTests/analysis/test_hole2.py +++ b/testsuite/MDAnalysisTests/analysis/test_hole2.py @@ -488,7 +488,7 @@ def test_over_order_parameters(self, hole): assert key == rmsd idx = np.argsort(op) - arr = np.array(list(hole.profiles.values())) + arr = np.array(list(hole.profiles.values()), dtype=object) for op_prof, arr_prof in zip(profiles.values(), arr[idx]): assert op_prof is arr_prof @@ -504,7 +504,7 @@ def test_over_order_parameters_file(self, hole, tmpdir): assert key == rmsd idx = np.argsort(op) - arr = np.array(list(hole.profiles.values())) + arr = np.array(list(hole.profiles.values()), dtype=object) for op_prof, arr_prof in zip(profiles.values(), arr[idx]): assert op_prof is arr_prof @@ -528,7 +528,7 @@ def test_over_order_parameters_frames(self, hole): idx = np.argsort(op[:n_frames]) values = list(hole.profiles.values())[:n_frames] - arr = np.array(values) + arr = np.array(values, dtype=object) for op_prof, arr_prof in zip(profiles.values(), arr[idx]): assert op_prof is arr_prof From 05e15b93cbaa5053a5c379d2c0893350b3d9efe2 Mon Sep 17 00:00:00 2001 From: Lily Wang <31115101+lilyminium@users.noreply.github.com> Date: Sun, 6 Dec 2020 11:54:35 +1100 Subject: [PATCH 2/7] Deploy docs via GitHub Actions CI (#3053) Fixes #3046. Deploys docs in gh-ci.yaml script and removes Travis --- .github/workflows/gh-ci.yaml | 53 +++++++++++++++++- maintainer/deploy_docs_via_travis.sh | 82 ---------------------------- 2 files changed, 51 insertions(+), 84 deletions(-) delete mode 100644 maintainer/deploy_docs_via_travis.sh diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml index a9fee810031..679dd97ec5f 100644 --- a/.github/workflows/gh-ci.yaml +++ b/.github/workflows/gh-ci.yaml @@ -19,6 +19,7 @@ env: MDA_PIP_MIN_DEPS: 'coveralls coverage<5 pytest-cov pytest-xdist' MDA_PIP_EXTRA_DEPS: 'duecredit parmed' + jobs: main_tests: if: "github.repository == 'MDAnalysis/mdanalysis'" @@ -211,9 +212,57 @@ jobs: - name: deploy_docs if: github.event_name != 'pull_request' + env: + GH_USER: github-actions + GH_EMAIL: "github-action@users.noreply.github.com" + GH_REPOSITORY: "github.com/${{ github.repository }}.git" + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + URL: https://docs.mdanalysis.org + run: | - # place the deploy call here - echo "Oh, maple syrup roast parsnips [Richard Gowers]" + # set up environment variables + # cannot execute bash to make variables in env section + # export URL for the Python script $UPDATE_JSON + export URL + export VERSION=$(cd package/MDAnalysis; python -c 'import version; print(version.__version__)') + UPDATE_JSON=$(pwd)/maintainer/update_json_stubs_sitemap.py + BRANCH="${GITHUB_REF#refs/heads/}" + + # the below turns off non-blocking as it causes large writes to stdout to fail + # (see https://github.com/travis-ci/travis-ci/issues/4704) + # commented out as this is not a problem with gh-actions + # python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' + cd package/doc/html/html + + # move docs into version subfolder + mkdir ../${VERSION} && mv * ../${VERSION} && mv ../${VERSION} $VERSION + + # set up git + REV=$(git rev-parse --short HEAD) + git init + git config user.name $GH_USER + git config user.email $GH_EMAIL + git remote add upstream "https://${GH_USER}:${GH_TOKEN}@${GH_REPOSITORY}" + git fetch --depth 50 upstream $BRANCH gh-pages + git reset upstream/gh-pages + + # redirects and copies + mkdir latest + python $UPDATE_JSON + touch . + touch .nojekyll + + git add -A ${VERSION}/ + git add .nojekyll versions.json *.xml *.html index.html latest + + for dirname in dev stable documentation_pages ; do + if [ -d $dirname ]; then git add $dirname; fi + done + + # check for anything to commit + # https://stackoverflow.com/questions/3878624/how-do-i-programmatically-determine-if-there-are-uncommited-changes + git diff-index --quiet HEAD -- || git commit -m "rebuilt html docs for version ${VERSION} from branch ${BRANCH} with sphinx at ${REV}" + git push -q upstream HEAD:gh-pages pylint_check: diff --git a/maintainer/deploy_docs_via_travis.sh b/maintainer/deploy_docs_via_travis.sh deleted file mode 100644 index 52c613e6b89..00000000000 --- a/maintainer/deploy_docs_via_travis.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# Deploying docs from travis-ci. -# See https://github.com/MDAnalysis/mdanalysis/issues/386 -# Script based on https://github.com/steveklabnik/automatically_update_github_pages_with_travis_example - -# Run this script from the top-level of the checked out git -# repository. A github OAuth token must be available in the evironment -# variable GH_TOKEN and is set up through the .travis.yml -# env:global:secure parameter (encrypted with travis-ci's public key)/ -# -# Additional environment variables set in .travis.yml -# GH_REPOSITORY repo to full from and push to -# GH_DOC_BRANCH branch from which the docs are built -# GIT_CI_USER name of the user to push docs as -# GIT_CI_EMAIL email of the user to push docs as -# MDA_DOCDIR path to the docdir from top of repo -# MAINTAIN_DIR path to maintainer/ -# VERSION version of MDAnalysis -# -# NOTE: If any of these environment variables are not set or -# empty then the script will exit with and error (-o nounset). - -set -o errexit -o nounset - -function die () { - local msg="$1" err=${2:-1} - echo "ERROR: $msg [$err]" - exit $err -} - -rev=$(git rev-parse --short HEAD) - -# the following tests should be superfluous because of -o nounset -test -n "${GH_TOKEN}" || die "GH_TOKEN is empty: need OAuth GitHub token to continue" 100 -test -n "${GH_REPOSITORY}" || die "GH_REPOSITORY must be set in .travis.yml" 100 -test -n "${MDA_DOCDIR}" || die "MDA_DOCDIR must be set in .travis.yml" 100 -test -n "${MAINTAIN_DIR}" || die "MAINTAIN_DIR must be set in .travis.yml" 100 -test -n "${VERSION}" || die "VERSION must be set in .travis.yml" 100 - - -cd ${MDA_DOCDIR} || die "Failed to 'cd ${MDA_DOCDIR}'. Run from the top level of the repository" - -# move into $version subdirectory -mkdir ../${VERSION} && mv * ../${VERSION} - -git init -git config user.name "${GIT_CI_USER}" -git config user.email "${GIT_CI_EMAIL}" - -mv ../${VERSION} $VERSION - -git remote add upstream "https://${GH_TOKEN}@${GH_REPOSITORY}" -git fetch --depth 50 upstream ${GH_DOC_BRANCH} gh-pages -git reset upstream/gh-pages - -# === REDIRECTS AND COPIES ==== -# home (index.html) redirects to stable/ -# latest (latest/index.html) redirects to most recent release -# dev/ is a copy of the dev docs with the highest number (so 2.0.0-dev instead of 1.0.1-dev) -# stable/ is a copy of the release docs with the highest number -mkdir latest -export URL="https://docs.mdanalysis.org" -python ${MAINTAIN_DIR}/update_json_stubs_sitemap.py -touch . -touch .nojekyll - -git add -A ${VERSION}/ -git add .nojekyll versions.json -git add index.html latest - -for dirname in dev stable documentation_pages ; do - if [ -d $dirname ]; then git add $dirname; fi -done - -git add *.xml *.html - -# check for anything to commit -# https://stackoverflow.com/questions/3878624/how-do-i-programmatically-determine-if-there-are-uncommited-changes -git diff-index --quiet HEAD -- || git commit -m "rebuilt html docs for version ${VERSION} from branch ${GH_DOC_BRANCH} with sphinx at ${rev}" -git push -q upstream HEAD:gh-pages - - From 62875af5cf3c3f3a567619942c2c06327a3ced41 Mon Sep 17 00:00:00 2001 From: Tyler Reddy Date: Mon, 19 Oct 2020 08:57:47 -0600 Subject: [PATCH 3/7] TST, CI: add ARM64 Graviton 2 to CI (#2956) * TST, CI: add ARM64 Graviton 2 to CI * start testing MDAnalysis on ARM64 using new AWS Graviton 2 architecture available in Travis CI * testing on my fork shows only two failures in the MDAnalysis test suite using a fairly minimal set of dependencies; we can probably either fix those or skip them and open an issue * the test failures are: `test_written_remarks_property` `TestEncoreClustering.test_clustering_three_ensembles_two_identical` * run time for this CI entry is about 23 minutes, which is solid for ARM; we save a lot of time using experimental upstream wheels in some cases and excluding (source builds of) dependencies where possible until the ARM64 binary wheel ecosystem matures a bit more * MAINT: ARM64 shims * adjust `CAffinityPropagation()` C-level function to use `double` instead of `float` in some strategic locations that allow the `test_clustering_three_ensembles_two_identical()` test to pass on gcc115 ARM64 machine * mark `test_written_remarks_property()` as a known failure on ARM64; it fails in Travis CI on that platform, but not on gcc115 ARM64 machine; that said, this test is already known to be flaky on Windows 32-bit (may depend on character settings on machine?) * MAINT: MR 2956 revisions * reduce optimization level of MDAnalysis builds on ARM64 to avoid problems with `ap.c` causing test failures on that architecture * revert any source changes to `ap.c` --- .disabled-travis.yml | 25 +++++++++++++++++++ package/setup.py | 11 ++++++-- .../MDAnalysisTests/formats/test_libdcd.py | 7 ++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/.disabled-travis.yml b/.disabled-travis.yml index 8638059e0ab..1569758168d 100644 --- a/.disabled-travis.yml +++ b/.disabled-travis.yml @@ -111,6 +111,31 @@ jobs: INSTALL_HOLE="false" CODECOV="true" SETUP_CMD="${PYTEST_FLAGS} --cov=MDAnalysis" + - os: linux + language: python + arch: arm64-graviton2 + python: + - "3.7" + dist: focal + virt: vm + group: edge + before_install: + - python -m pip install cython numpy + # special test SciPy wheel for ARM64: + - python -m pip install https://anaconda.org/multibuild-wheels-staging/scipy/1.6.0.dev0+a240c17/download/scipy-1.6.0.dev0+a240c17-cp37-cp37m-manylinux2014_aarch64.whl + - python -m pip install --no-build-isolation hypothesis matplotlib pytest pytest-cov pytest-xdist tqdm + install: + - cd package + - python setup.py install + - cd ../testsuite + - python setup.py install + - cd .. + script: + - cd testsuite + - python -m pytest ./MDAnalysisTests --disable-pytest-warnings -n 8 -rsx --cov=MDAnalysis + after_success: + - echo "Override this stage for ARM64" + allow_failures: - env: NUMPY_VERSION=dev EVENT_TYPE="cron" diff --git a/package/setup.py b/package/setup.py index e48e12c52e2..c5fc43675d9 100755 --- a/package/setup.py +++ b/package/setup.py @@ -268,8 +268,15 @@ def extensions(config): use_cython = config.get('use_cython', default=not is_release) use_openmp = config.get('use_openmp', default=True) - extra_compile_args = ['-std=c99', '-ffast-math', '-O3', '-funroll-loops', - '-fsigned-zeros'] # see #2722 + if platform.machine() == 'aarch64': + # reduce optimization level for ARM64 machines + # because of issues with test failures sourcing to: + # MDAnalysis/analysis/encore/clustering/src/ap.c + extra_compile_args = ['-std=c99', '-ffast-math', '-O1', '-funroll-loops', + '-fsigned-zeros'] + else: + extra_compile_args = ['-std=c99', '-ffast-math', '-O3', '-funroll-loops', + '-fsigned-zeros'] # see #2722 define_macros = [] if config.get('debug_cflags', default=False): extra_compile_args.extend(['-Wall', '-pedantic']) diff --git a/testsuite/MDAnalysisTests/formats/test_libdcd.py b/testsuite/MDAnalysisTests/formats/test_libdcd.py index 0028a80bd3b..240aee1a74e 100644 --- a/testsuite/MDAnalysisTests/formats/test_libdcd.py +++ b/testsuite/MDAnalysisTests/formats/test_libdcd.py @@ -22,6 +22,7 @@ import sys import string import struct +import platform import hypothesis.strategies as strategies from hypothesis import example, given @@ -317,8 +318,10 @@ def write_dcd(in_name, out_name, remarks='testing', header=None): f_out.write(xyz=frame.xyz, box=frame.unitcell) -@pytest.mark.xfail(os.name == 'nt' and sys.maxsize <= 2**32, - reason="occasional fail on 32-bit windows") +@pytest.mark.xfail((os.name == 'nt' + and sys.maxsize <= 2**32) or + platform.machine() == 'aarch64', + reason="occasional fail on 32-bit windows and ARM") @given(remarks=strategies.text( alphabet=string.printable, min_size=0, max_size=239)) # handle the printable ASCII strings From de022c410376ba4255f15feea662d6fa13636130 Mon Sep 17 00:00:00 2001 From: Lily Wang <31115101+lilyminium@users.noreply.github.com> Date: Tue, 20 Oct 2020 14:45:29 +1100 Subject: [PATCH 4/7] Re-write URLs in sitemap.xml (#2994) Fixes #2993 Redirect URLs in stable/sitemap.xml and dev/sitemap.xml to correct locations From 8851a2e68ca94f508374a2cf19e979a4b8b192d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bouysset?= Date: Fri, 30 Oct 2020 20:50:21 +0100 Subject: [PATCH 5/7] ParmEdConverter: fix NameError when catching NoDataError (#2953) * fix NameError when catching NoDataError * add test + missing imports --- package/MDAnalysis/coordinates/ParmEd.py | 3 +++ testsuite/MDAnalysisTests/coordinates/test_parmed.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/package/MDAnalysis/coordinates/ParmEd.py b/package/MDAnalysis/coordinates/ParmEd.py index c35b16103f0..dfb996903bd 100644 --- a/package/MDAnalysis/coordinates/ParmEd.py +++ b/package/MDAnalysis/coordinates/ParmEd.py @@ -74,10 +74,13 @@ """ from __future__ import absolute_import import functools +import itertools +import warnings from . import base from ..topology.tables import SYMB2Z from ..core.universe import Universe +from ..exceptions import NoDataError class ParmEdReader(base.SingleFrameReaderBase): diff --git a/testsuite/MDAnalysisTests/coordinates/test_parmed.py b/testsuite/MDAnalysisTests/coordinates/test_parmed.py index a1fe4e44326..5dd350e86fd 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_parmed.py +++ b/testsuite/MDAnalysisTests/coordinates/test_parmed.py @@ -206,6 +206,17 @@ def test_equivalent_dihedrals(self, ref, roundtrip): p.improper == q.improper and p.ignore_end == q.ignore_end) for q in original) + def test_missing_attr(self): + n_atoms = 10 + u = mda.Universe.empty(n_atoms) + u.add_TopologyAttr("resid", [1]) + u.add_TopologyAttr("segid", ["DUM"]) + u.add_TopologyAttr("mass", [1] * n_atoms) + with pytest.warns(UserWarning, + match="Supplied AtomGroup was missing the following " + "attributes"): + # should miss names and resnames + u.atoms.convert_to("PARMED") class BaseTestParmEdConverterSubset(BaseTestParmEdConverter): From 218022aa5991753efc9e67ba307409210ecd5cac Mon Sep 17 00:00:00 2001 From: Summersnow Date: Thu, 19 Nov 2020 10:57:47 +0800 Subject: [PATCH 6/7] Improve the performance of ParmEd converter. (Fix #3028) (#3029) Fixes #3028 * Improves the performance of the ParmEd converter by using a dictionary lookup for the atomgroup to universe index mapping. --- package/AUTHORS | 1 + package/CHANGELOG | 3 ++- package/MDAnalysis/coordinates/ParmEd.py | 6 ++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/package/AUTHORS b/package/AUTHORS index 401bd935fca..7ebc3b8a96c 100644 --- a/package/AUTHORS +++ b/package/AUTHORS @@ -149,6 +149,7 @@ Chronological list of authors - Marcello Sega - Nicholas Craven - Ramon Crehuet + - Haochuan Chen External code ------------- diff --git a/package/CHANGELOG b/package/CHANGELOG index 3dfb362bacb..5e252f2b1af 100644 --- a/package/CHANGELOG +++ b/package/CHANGELOG @@ -15,7 +15,7 @@ The rules for this file: ------------------------------------------------------------------------------ ??/??/20 richardjgowers, IAlibay, orbeckst, tylerjereddy, jbarnoud, yuxuanzhuang, lilyminium, VOD555, p-j-smith, bieniekmateusz, - calcraven, ianmkenney, rcrehuet, manuel.nuno.melo + calcraven, ianmkenney, rcrehuet, manuel.nuno.melo, hanatok * 1.0.1 @@ -50,6 +50,7 @@ Fixes (Issue #2934) Enhancements + * Improved performance of the ParmEd converter (Issue #3028, PR #3029) * Improved performances when parsing TPR files (PR #2804) Changes (not affecting users) diff --git a/package/MDAnalysis/coordinates/ParmEd.py b/package/MDAnalysis/coordinates/ParmEd.py index dfb996903bd..b2eb92d9c3f 100644 --- a/package/MDAnalysis/coordinates/ParmEd.py +++ b/package/MDAnalysis/coordinates/ParmEd.py @@ -133,7 +133,7 @@ def _read_first_frame(self): } def get_indices_from_subset(i, atomgroup=None, universe=None): - return atomgroup.index(universe.atoms[i]) + return atomgroup[universe.atoms[i]] class ParmEdConverter(base.ConverterBase): """Convert MDAnalysis AtomGroup or Universe to ParmEd :class:`~parmed.structure.Structure`. @@ -263,8 +263,10 @@ def convert(self, obj): struct.box = None if hasattr(ag_or_ts, 'universe'): + atomgroup = {atom: index for index, + atom in enumerate(list(ag_or_ts))} get_atom_indices = functools.partial(get_indices_from_subset, - atomgroup=list(ag_or_ts), + atomgroup=atomgroup, universe=ag_or_ts.universe) else: get_atom_indices = lambda x: x From e79c80ca5fc9117d80879d996a3f61597f4d3cab Mon Sep 17 00:00:00 2001 From: Lily Wang Date: Tue, 8 Dec 2020 11:34:48 +1100 Subject: [PATCH 7/7] attempt py2... --- .github/workflows/gh-ci.yaml | 53 +++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml index 679dd97ec5f..f66f5b65a26 100644 --- a/.github/workflows/gh-ci.yaml +++ b/.github/workflows/gh-ci.yaml @@ -15,24 +15,43 @@ defaults: env: MDA_CONDA_MIN_DEPS: "pip pytest mmtf-python biopython networkx cython matplotlib scipy griddataformats hypothesis gsd codecov" - MDA_CONDA_EXTRA_DEPS: "seaborn>=0.7.0 clustalw=2.1 netcdf4 scikit-learn joblib>=0.12 chemfiles tqdm>=4.43.0 tidynamics>=1.0.0 rdkit>=2020.03.1 h5py==2.10.0" + MDA_CONDA_EXTRA_DEPS: "seaborn" + MDA_CONDA_PY3_DEPS: "netcdf4 scikit-learn joblib>=0.12 clustalw=2.1 chemfiles mock" + MDA_CONDA_PY27_DEPS: "clustalw=2.1 netcdf4=1.3.1 six=1.15.0 funcsigs<=1.0.2 mock<=3.0.5" + MDA_CONDA_PY35_DEPS: "netcdf4 scikit-learn joblib>=0.12 clustalw=2.1 mock<=3.0.5" MDA_PIP_MIN_DEPS: 'coveralls coverage<5 pytest-cov pytest-xdist' MDA_PIP_EXTRA_DEPS: 'duecredit parmed' - + MDA_PIP_PY27_DEPS: "setuptools<45.0.0 chemfiles scikit-learn joblib>=0.12" + MDA_PIP_PY3_DEPS: "" + MDA_PIP_PY35_DEPS: "" jobs: main_tests: - if: "github.repository == 'MDAnalysis/mdanalysis'" runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ] python-version: [3.6, 3.7, 3.8] - run_type: [FULL, ] + extra_deps: ["EXTRA PY3", ] install_hole: [true, ] codecov: [true, ] + conda-version: ["latest", ] include: + - name: python-2.7 + os: ubuntu-latest + python-version: 2.7 + install_hole: true + codecov: true + numpy: numpy=1.16 + extra_deps: "EXTRA PY27" + conda-version: 4.7.10 + - name: python-3.5 + os: ubuntu-latest + python-version: 3.5 + install_hole: true + codecov: true + extra_deps: "EXTRA PY35" - name: macOS os: macOS-latest python-version: 3.7 @@ -42,20 +61,20 @@ jobs: - name: minimal-ubuntu os: ubuntu-latest python-version: 3.6 - run_type: MIN + extra_deps: "" install_hole: false codecov: true - name: numpy_min os: ubuntu-latest python-version: 3.6 - run_type: FULL + extra_deps: "" install_hole: false codecov: false numpy: numpy=1.16.0 - name: asv_check os: ubuntu-latest python-version: 3.7 - run_type: FULL + extra_deps: "EXTRA PY3" install_hole: false codecov: false env: @@ -99,21 +118,29 @@ jobs: python-version: ${{ matrix.python-version }} auto-update-conda: true channel-priority: flexible - channels: biobuilds, conda-forge + channels: biobuilds, conda-forge, defaults add-pip-as-python-dependency: true + miniconda-version: ${{ matrix.conda-version }} # TODO: mamba causes pip to segfault, switch when fixed #mamba-version: "*" architecture: x64 - name: install_deps - env: - MDA_CONDA_FULL_DEPS: "${{ env.MDA_CONDA_MIN_DEPS }} ${{ env.MDA_CONDA_EXTRA_DEPS }}" - MDA_PIP_FULL_DEPS: "${{ env.MDA_PIP_MIN_DEPS }} ${{ env.MDA_PIP_EXTRA_DEPS }}" run: | # NOTE: vars need to be re-assigned # NOTE: set matrix.numpy to pin to a specific numpy version - conda_deps="${{ matrix.numpy }} ${MDA_CONDA_${{ matrix.run_type }}_DEPS}" - pip_deps=${MDA_PIP_${{ matrix.run_type }}_DEPS} + conda_deps="${{ matrix.numpy }} ${MDA_CONDA_MIN_DEPS}" + pip_deps="${MDA_PIP_MIN_DEPS}" + + for dep in ${{ matrix.extra_deps }}; do + conda_var="MDA_CONDA_${dep}_DEPS" + pip_var="MDA_PIP_${dep}_DEPS" + echo "${!conda_var}" + conda_deps="$conda_deps ${!conda_var}" + echo $conda_deps + pip_deps="$pip_deps ${!pip_var}" + done + conda install ${conda_deps} pip install ${pip_deps}