diff --git a/.travis.yml.future b/.github/workflows/cypress_ui.yml.future similarity index 74% rename from .travis.yml.future rename to .github/workflows/cypress_ui.yml.future index 8bd747625e4..b38ae2f9558 100644 --- a/.travis.yml.future +++ b/.github/workflows/cypress_ui.yml.future @@ -1,16 +1,17 @@ +############################################################################### +# +# THIS IS AN OLD TRAVIS-CI.ORG JOB FILE +# To be used with Github Actions, it would be necessary to refactor it. +# Keeping it as the future example it has been before. +# See also #5846 +# +############################################################################### + services: - docker jobs: include: - # Execute java unit- and integration tests - - stage: test - language: java - jdk: - - oraclejdk8 - script: mvn -DcompilerArgument=-Xlint:unchecked test -P all-unit-tests - after_success: mvn jacoco:report coveralls:report - # Execute Cypress for UI testing # see https://docs.cypress.io/guides/guides/continuous-integration.html - stage: test diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml new file mode 100644 index 00000000000..916f0e7ecbb --- /dev/null +++ b/.github/workflows/maven_unit_test.yml @@ -0,0 +1,45 @@ +name: Maven Unit Tests + +on: + push: + paths: + - "**.java" + pull_request: + paths: + - "**.java" + +jobs: + unittest: + name: (JDK ${{ matrix.jdk }} / ${{ matrix.os }}) Unit Tests + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + jdk: [ '11' ] + #include: + # - os: ubuntu-latest + # jdk: '16' + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.jdk }} + distribution: 'adopt' + - name: Cache Maven packages + uses: actions/cache@v2 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + - name: Build with Maven + run: mvn -DcompilerArgument=-Xlint:unchecked -P all-unit-tests clean test + - name: Maven Code Coverage + env: + CI_NAME: github + BRANCH_NAME_OR_REF: ${{ github.head_ref || github.ref }} + CI_BUILD_NUMBER: ${{ github.run_id }} + CI_BUILD_URL: https://github.com/${{ github.repository }}/commit/${{ github.event.after }}/checks + COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }} + run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${{ secrets.COVERALLS_TOKEN }} \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 67de6619add..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,5 +0,0 @@ -language: java -jdk: - - openjdk11 -script: mvn -DcompilerArgument=-Xlint:unchecked test -P all-unit-tests -after_success: mvn jacoco:report coveralls:report diff --git a/README.md b/README.md index 3f3b8c2de90..6fd11374353 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/) [![API Test Coverage](https://img.shields.io/jenkins/coverage/jacoco?jobUrl=https%3A%2F%2Fjenkins.dataverse.org%2Fjob%2FIQSS-dataverse-develop&label=API%20Test%20Coverage)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/ws/target/coverage-it/index.html) -[![Unit Test Status](https://img.shields.io/travis/IQSS/dataverse?label=Unit%20Test%20Status)](https://travis-ci.org/IQSS/dataverse) +[![Unit Test Status](https://github.com/IQSS/dataverse/actions/workflows/maven_unit_test.yml/badge.svg?branch=develop)](https://github.com/IQSS/dataverse/actions/workflows/maven_unit_test.yml) [![Unit Test Coverage](https://img.shields.io/coveralls/github/IQSS/dataverse?label=Unit%20Test%20Coverage)](https://coveralls.io/github/IQSS/dataverse?branch=develop) [![Guides Build Status](https://github.com/IQSS/dataverse/actions/workflows/guides_build_sphinx.yml/badge.svg)](https://github.com/IQSS/dataverse/actions/workflows/guides_build_sphinx.yml) diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index e7bc6b2fed5..7bde4055e33 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -37,11 +37,9 @@ A unit test should execute an operation of your code in a controlled fashion. Yo Unit Test Automation Overview ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -We use a variety of tools to write, execute, and measure the code coverage of unit tests, including Maven, JUnit, Jacoco, GitHub, Travis, and Coveralls. We'll explain the role of each tool below, but here's an overview of what you can expect from the automation we've set up. +We use a variety of tools to write, execute, and measure the code coverage of unit tests, including Maven, JUnit, Jacoco, GitHub, and Coveralls. We'll explain the role of each tool below, but here's an overview of what you can expect from the automation we've set up. -As you prepare to make a pull request, as described in the :doc:`version-control` section, you will be working on a new branch you create from the "develop" branch. Let's say your branch is called ``1012-private-url``. As you work, you are constantly invoking Maven to build the war file. When you do a "clean and build" in Netbeans, Maven runs all the unit tests (anything ending with ``Test.java``) and the runs the results through a tool called Jacoco that calculates code coverage. When you push your branch to GitHub and make a pull request, a web service called Travis CI runs Maven and Jacoco on your branch and pushes the results to Coveralls, which is a web service that tracks changes to code coverage over time. - -To make this more concrete, observe that https://github.com/IQSS/dataverse/pull/3111 has comments from a GitHub user called ``coveralls`` saying things like "Coverage increased (+0.5%) to 5.547% when pulling dd6ceb1 on 1012-private-url into be5b26e on develop." Clicking on the comment should lead you to a URL such as https://coveralls.io/builds/7013870 which shows how code coverage has gone up or down. That page links to a page such as https://travis-ci.org/IQSS/dataverse/builds/144840165 which shows the build on the Travis side that pushed the results to Coveralls. Note that we have configured Coveralls to not mark small decreases in code coverage as a failure. +As you prepare to make a pull request, as described in the :doc:`version-control` section, you will be working on a new branch you create from the "develop" branch. Let's say your branch is called ``1012-private-url``. As you work, you are constantly invoking Maven to build the war file. When you do a "clean and build" in Netbeans, Maven runs all the unit tests (anything ending with ``Test.java``) and then runs the results through a tool called Jacoco that calculates code coverage. When you push your branch to GitHub and make a pull request, GitHub Actions runs Maven and Jacoco on your branch and pushes the results to Coveralls, which is a web service that tracks changes to code coverage over time. Note that we have configured Coveralls to not mark small decreases in code coverage as a failure. You can find the Coveralls reports at https://coveralls.io/github/IQSS/dataverse The main takeaway should be that we care about unit testing enough to measure the changes to code coverage over time using automation. Now let's talk about how you can help keep our code coverage up by writing unit tests with JUnit. @@ -102,12 +100,10 @@ In addition, there is a writeup on "The Testable Command" at https://github.com/ Running Non-Essential (Excluded) Unit Tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You should be aware that some unit tests have been deemed "non-essential" and have been annotated with ``@Category(NonEssentialTests.class)`` and are excluded from the "dev" Maven profile, which is the default profile. All unit tests (that have not been annotated with ``@Ignore``), including these non-essential tests, are run from continuous integration systems such as Jenkins and Travis CI with the following ``mvn`` command that invokes a non-default profile: +You should be aware that some unit tests have been deemed "non-essential" and have been annotated with ``@Category(NonEssentialTests.class)`` and are excluded from the "dev" Maven profile, which is the default profile. All unit tests (that have not been annotated with ``@Ignore``), including these non-essential tests, are run from continuous integration systems such as Jenkins and GitHub Actions with the following ``mvn`` command that invokes a non-default profile: ``mvn test -P all-unit-tests`` -Typically https://travis-ci.org/IQSS/dataverse will show a higher number of unit tests executed because it uses the profile above. - Generally speaking, unit tests have been flagged as non-essential because they are slow or because they require an Internet connection. You should not feel obligated to run these tests continuously but you can use the ``mvn`` command above to run them. To iterate on the unit test in Netbeans and execute it with "Run -> Test File", you must temporarily comment out the annotation flagging the test as non-essential. Integration Tests @@ -393,12 +389,12 @@ The script requires a file called ``files.txt`` to operate and database IDs for Continuous Integration ---------------------- -The Dataverse Project currently makes use of two Continuous Integration platforms, Travis and Jenkins. - -Travis builds are configured via :download:`.travis.yml <../../../../.travis.yml>` and a `GitHub webhook `; build output is viewable at https://travis-ci.org/IQSS/dataverse/builds +The Dataverse Project currently makes use of two Continuous Integration platforms, Jenkins and GitHub Actions. Our Jenkins config is a work in progress and may be viewed at https://github.com/IQSS/dataverse-jenkins/ A corresponding GitHub webhook is required. Build output is viewable at https://jenkins.dataverse.org/ +GitHub Actions jobs can be found in ``.github/workflows``. + As always, pull requests to improve our continuous integration configurations are welcome. Enhance build time by caching dependencies @@ -475,7 +471,6 @@ Future Work on Integration Tests - Automate testing of dataverse-client-python: https://github.com/IQSS/dataverse-client-python/issues/10 - Work with @leeper on testing the R client: https://github.com/IQSS/dataverse-client-r - Review and attempt to implement "API Test Checklist" from @kcondon at https://docs.google.com/document/d/199Oq1YwQ4pYCguaeW48bIN28QAitSk63NbPYxJHCCAE/edit?usp=sharing -- Attempt to use @openscholar approach for running integration tests using Travis https://github.com/openscholar/openscholar/blob/SCHOLAR-3.x/.travis.yml (probably requires using Ubuntu rather than CentOS) - Generate code coverage reports for **integration** tests: https://github.com/pkainulainen/maven-examples/issues/3 and http://www.petrikainulainen.net/programming/maven/creating-code-coverage-reports-for-unit-and-integration-tests-with-the-jacoco-maven-plugin/ - Consistent logging of API Tests. Show test name at the beginning and end and status codes returned. - expected passing and known/expected failing integration tests: https://github.com/IQSS/dataverse/issues/4438 @@ -488,15 +483,14 @@ Browser-Based Testing Installation Testing ~~~~~~~~~~~~~~~~~~~~ -- Run `vagrant up` on a server to test the installer: http://guides.dataverse.org/en/latest/developers/tools.html#vagrant . We haven't been able to get this working in Travis: https://travis-ci.org/IQSS/dataverse/builds/96292683 . Perhaps it would be possible to use AWS as a provider from Vagrant judging from https://circleci.com/gh/critical-alert/circleci-vagrant/6 . -- Work with @lwo to automate testing of https://github.com/IQSS/dataverse-puppet . Consider using Travis: https://github.com/IQSS/dataverse-puppet/issues/10 -- Work with @donsizemore to automate testing of https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible with Travis or similar. +- Run `vagrant up` on a server to test the installer +- Work with @donsizemore to automate testing of https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible Future Work on Load/Performance Testing ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Clean up and copy stress tests code, config, and docs into main repo from https://github.com/IQSS/dataverse-helper-scripts/tree/master/src/stress_tests -- Marcel Duran created a command-line wrapper for the WebPagetest API that can be used to test performance in your continuous integration pipeline (TAP, Jenkins, Travis-CI, etc): https://github.com/marcelduran/webpagetest-api/wiki/Test-Specs#jenkins-integration +- Marcel Duran created a command-line wrapper for the WebPagetest API that can be used to test performance in your continuous integration pipeline (TAP, Jenkins, etc.): https://github.com/marcelduran/webpagetest-api/wiki/Test-Specs#jenkins-integration - Create top-down checklist, building off the "API Test Coverage" spreadsheet at https://github.com/IQSS/dataverse/issues/3358#issuecomment-256400776 Future Work on Accessibility Testing diff --git a/doc/theTestableCommand/TheTestableCommand.md b/doc/theTestableCommand/TheTestableCommand.md index e60faa313f0..5a9fc259d4a 100644 --- a/doc/theTestableCommand/TheTestableCommand.md +++ b/doc/theTestableCommand/TheTestableCommand.md @@ -21,7 +21,7 @@ While they can't replace end-to-end tests, unit tests are a great way to validat Because unit tests are easy to create (Java only, no configuration needed) and quick to run, it is possible to write many of them, such that many aspects of the code are tested. Normally, a single unit test would test a single use case of the unit. This way, when a unit test fails, the failure describes exactly what part stopped functioning. Other unit tests are not blocked by the failure, and so by running the entire test suite, the developer can get a good overview of which parts are broken and which parts are functioning well. -Because unit tests are easy to execute, it is recommended to get in the habit of running them prior to committing code changes to the repository. These tests are also integrated into Dataverse's automatic build processes (on [Travis-ci](https://travis-ci.org/IQSS/dataverse)). A failed test halts the build. Dataverse's build process also collects data about code coverage during the unit tests, using [Coveralls](https://coveralls.io/github/IQSS/dataverse). While code coverage is a problematic measure for Java EE applications (and has some inherent problems as well), generally speaking larger coverage means better testing. +Because unit tests are easy to execute, it is recommended to get in the habit of running them prior to committing code changes to the repository. These tests are also integrated into Dataverse's automatic build processes. A failed test halts the build. Dataverse's build process also collects data about code coverage during the unit tests, using [Coveralls](https://coveralls.io/github/IQSS/dataverse). While code coverage is a problematic measure for Java EE applications (and has some inherent problems as well), generally speaking larger coverage means better testing. Unit Testing of application logic in Java EE applications is normally hard to do, as the application logic lives in the service beans, which rely on dependency injections. Writing unit tests for service beans is possible, but as it involves a test container, and a persistent context (read: in-memory database) these unit tests are not very unit-y. diff --git a/pom.xml b/pom.xml index ca2d3570956..d3aaf2fc90e 100644 --- a/pom.xml +++ b/pom.xml @@ -783,7 +783,7 @@ org.eluder.coveralls coveralls-maven-plugin - 4.0.0 + 4.3.0 javax.xml.bind