From 975f83effed7d9d793c8e9c85badcb72042a39a1 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Sat, 24 Jan 2026 07:29:36 +1000 Subject: [PATCH 1/2] Remove -x from mpl pytest runs --- .github/workflows/build-ultraplot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-ultraplot.yml b/.github/workflows/build-ultraplot.yml index 9ebab8d0e..c4c782dfb 100644 --- a/.github/workflows/build-ultraplot.yml +++ b/.github/workflows/build-ultraplot.yml @@ -102,7 +102,7 @@ jobs: # Generate the baseline images and hash library python -c "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" - pytest -x -W ignore \ + pytest -W ignore \ --mpl-generate-path=./ultraplot/tests/baseline/ \ --mpl-default-style="./ultraplot.yml"\ ultraplot/tests @@ -120,7 +120,7 @@ jobs: mkdir -p results python -c "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" - pytest -x -W ignore \ + pytest -W ignore \ --mpl \ --mpl-baseline-path=./ultraplot/tests/baseline \ --mpl-results-path=./results/ \ From a612e5074bb0d1997a00327420651e44371a4c21 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Sat, 24 Jan 2026 08:39:01 +1000 Subject: [PATCH 2/2] ci: add test impact selection --- .github/workflows/build-ultraplot.yml | 58 ++++++++++++++---- .github/workflows/main.yml | 59 ++++++++++++++++++- .github/workflows/test-map.yml | 46 +++++++++++++++ tools/ci/build_test_map.py | 69 ++++++++++++++++++++++ tools/ci/select_tests.py | 85 +++++++++++++++++++++++++++ 5 files changed, 304 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/test-map.yml create mode 100644 tools/ci/build_test_map.py create mode 100644 tools/ci/select_tests.py diff --git a/.github/workflows/build-ultraplot.yml b/.github/workflows/build-ultraplot.yml index c4c782dfb..9899017df 100644 --- a/.github/workflows/build-ultraplot.yml +++ b/.github/workflows/build-ultraplot.yml @@ -8,6 +8,14 @@ on: matplotlib-version: required: true type: string + test-mode: + required: false + type: string + default: full + test-nodeids: + required: false + type: string + default: "" env: LC_ALL: en_US.UTF-8 @@ -21,6 +29,9 @@ jobs: defaults: run: shell: bash -el {0} + env: + TEST_MODE: ${{ inputs.test-mode }} + TEST_NODEIDS: ${{ inputs.test-nodeids }} steps: - uses: actions/checkout@v6 with: @@ -43,7 +54,11 @@ jobs: - name: Test Ultraplot run: | - pytest -n auto --cov=ultraplot --cov-branch --cov-report term-missing --cov-report=xml ultraplot + if [ "${TEST_MODE}" = "selected" ] && [ -n "${TEST_NODEIDS}" ]; then + pytest -n auto --cov=ultraplot --cov-branch --cov-report term-missing --cov-report=xml ${TEST_NODEIDS} + else + pytest -n auto --cov=ultraplot --cov-branch --cov-report term-missing --cov-report=xml ultraplot + fi - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 @@ -56,6 +71,8 @@ jobs: runs-on: ubuntu-latest env: IS_PR: ${{ github.event_name == 'pull_request' }} + TEST_MODE: ${{ inputs.test-mode }} + TEST_NODEIDS: ${{ inputs.test-nodeids }} defaults: run: shell: bash -el {0} @@ -102,10 +119,17 @@ jobs: # Generate the baseline images and hash library python -c "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" - pytest -W ignore \ - --mpl-generate-path=./ultraplot/tests/baseline/ \ - --mpl-default-style="./ultraplot.yml"\ - ultraplot/tests + if [ "${TEST_MODE}" = "selected" ] && [ -n "${TEST_NODEIDS}" ]; then + pytest -W ignore \ + --mpl-generate-path=./ultraplot/tests/baseline/ \ + --mpl-default-style="./ultraplot.yml" \ + ${TEST_NODEIDS} + else + pytest -W ignore \ + --mpl-generate-path=./ultraplot/tests/baseline/ \ + --mpl-default-style="./ultraplot.yml" \ + ultraplot/tests + fi # Return to the PR branch for the rest of the job if [ -n "${{ github.event.pull_request.base.sha }}" ]; then @@ -120,13 +144,23 @@ jobs: mkdir -p results python -c "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')" - pytest -W ignore \ - --mpl \ - --mpl-baseline-path=./ultraplot/tests/baseline \ - --mpl-results-path=./results/ \ - --mpl-generate-summary=html \ - --mpl-default-style="./ultraplot.yml" \ - ultraplot/tests + if [ "${TEST_MODE}" = "selected" ] && [ -n "${TEST_NODEIDS}" ]; then + pytest -W ignore \ + --mpl \ + --mpl-baseline-path=./ultraplot/tests/baseline \ + --mpl-results-path=./results/ \ + --mpl-generate-summary=html \ + --mpl-default-style="./ultraplot.yml" \ + ${TEST_NODEIDS} + else + pytest -W ignore \ + --mpl \ + --mpl-baseline-path=./ultraplot/tests/baseline \ + --mpl-results-path=./results/ \ + --mpl-generate-summary=html \ + --mpl-default-style="./ultraplot.yml" \ + ultraplot/tests + fi # Return the html output of the comparison even if failed - name: Upload comparison failures diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cc8b1b68..c035214ed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,6 +19,60 @@ jobs: python: - 'ultraplot/**' + select-tests: + runs-on: ubuntu-latest + needs: + - run-if-changes + if: always() && needs.run-if-changes.outputs.run == 'true' + outputs: + mode: ${{ steps.select.outputs.mode }} + tests: ${{ steps.select.outputs.tests }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Prepare workspace + run: mkdir -p .ci + + - name: Restore test map cache + id: restore-map + uses: actions/cache/restore@v4 + with: + path: .ci/test-map.json + key: test-map-${{ github.event.pull_request.base.sha }} + restore-keys: | + test-map- + + - name: Select impacted tests + id: select + run: | + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "mode=full" >> $GITHUB_OUTPUT + echo "tests=" >> $GITHUB_OUTPUT + exit 0 + fi + + git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} > .ci/changed.txt + + python tools/ci/select_tests.py \ + --map .ci/test-map.json \ + --changed-files .ci/changed.txt \ + --output .ci/selection.json \ + --always-full 'pyproject.toml' \ + --always-full 'environment.yml' \ + --always-full 'ultraplot/__init__.py' \ + --ignore 'docs/**' \ + --ignore 'README.rst' + + python - <<'PY' > .ci/selection.out + import json + data = json.load(open(".ci/selection.json", "r", encoding="utf-8")) + print(f"mode={data['mode']}") + print("tests=" + " ".join(data.get("tests", []))) + PY + cat .ci/selection.out >> $GITHUB_OUTPUT + get-versions: runs-on: ubuntu-latest needs: @@ -121,7 +175,8 @@ jobs: needs: - get-versions - run-if-changes - if: always() && needs.run-if-changes.outputs.run == 'true' && needs.get-versions.result == 'success' + - select-tests + if: always() && needs.run-if-changes.outputs.run == 'true' && needs.get-versions.result == 'success' && needs.select-tests.result == 'success' strategy: matrix: python-version: ${{ fromJson(needs.get-versions.outputs.python-versions) }} @@ -134,6 +189,8 @@ jobs: with: python-version: ${{ matrix.python-version }} matplotlib-version: ${{ matrix.matplotlib-version }} + test-mode: ${{ needs.select-tests.outputs.mode }} + test-nodeids: ${{ needs.select-tests.outputs.tests }} build-success: needs: diff --git a/.github/workflows/test-map.yml b/.github/workflows/test-map.yml new file mode 100644 index 000000000..f5c23e1e5 --- /dev/null +++ b/.github/workflows/test-map.yml @@ -0,0 +1,46 @@ +name: Build Test Map +on: + push: + branches: [main] + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + +jobs: + build-map: + runs-on: ubuntu-latest + timeout-minutes: 90 + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: mamba-org/setup-micromamba@v2.0.7 + with: + environment-file: ./environment.yml + init-shell: bash + create-args: >- + --verbose + python=3.11 + matplotlib=3.9 + cache-environment: true + cache-downloads: false + + - name: Build Ultraplot + run: | + pip install --no-build-isolation --no-deps . + + - name: Generate test coverage map + run: | + mkdir -p .ci + pytest -n auto --cov=ultraplot --cov-branch --cov-context=test --cov-report= ultraplot + python tools/ci/build_test_map.py --coverage-file .coverage --output .ci/test-map.json --root . + + - name: Cache test map + uses: actions/cache@v4 + with: + path: .ci/test-map.json + key: test-map-${{ github.sha }} diff --git a/tools/ci/build_test_map.py b/tools/ci/build_test_map.py new file mode 100644 index 000000000..3708e73e1 --- /dev/null +++ b/tools/ci/build_test_map.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +from datetime import datetime, timezone +from pathlib import Path + + +def build_map(coverage_file: str, repo_root: str) -> dict[str, list[str]]: + try: + from coverage import Coverage + except Exception as exc: # pragma: no cover - diagnostic path + raise SystemExit( + f"coverage.py is required to build the test map: {exc}" + ) from exc + + cov = Coverage(data_file=coverage_file) + cov.load() + data = cov.get_data() + + files_map: dict[str, set[str]] = {} + for filename in data.measured_files(): + if not filename: + continue + rel = os.path.relpath(filename, repo_root) + if rel.startswith(".."): + continue + try: + contexts_by_line = data.contexts_by_lineno(filename) + except Exception: + continue + + contexts = set() + for ctxs in contexts_by_line.values(): + if ctxs: + contexts.update(ctxs) + if contexts: + files_map[rel] = contexts + + return {path: sorted(contexts) for path, contexts in files_map.items()} + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Build a test impact map from coverage contexts." + ) + parser.add_argument("--coverage-file", default=".coverage") + parser.add_argument("--output", required=True) + parser.add_argument("--root", default=".") + args = parser.parse_args() + + repo_root = os.path.abspath(args.root) + mapping = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "files": build_map(args.coverage_file, repo_root), + } + + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as f: + json.dump(mapping, f, indent=2, sort_keys=True) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/ci/select_tests.py b/tools/ci/select_tests.py new file mode 100644 index 000000000..46565b71d --- /dev/null +++ b/tools/ci/select_tests.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +from fnmatch import fnmatch +from pathlib import Path + + +def load_map(path: str) -> dict[str, list[str]] | None: + map_path = Path(path) + if not map_path.is_file(): + return None + with map_path.open("r", encoding="utf-8") as f: + data = json.load(f) + return data.get("files", {}) + + +def read_changed_files(path: str) -> list[str]: + changed_path = Path(path) + if not changed_path.is_file(): + return [] + return [ + line.strip() + for line in changed_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def matches_any(path: str, patterns: list[str]) -> bool: + return any(fnmatch(path, pattern) for pattern in patterns) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Select impacted pytest nodeids from a test map." + ) + parser.add_argument("--map", dest="map_path", required=True) + parser.add_argument("--changed-files", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--always-full", action="append", default=[]) + parser.add_argument("--ignore", action="append", default=[]) + parser.add_argument("--source-prefix", default="ultraplot/") + parser.add_argument("--tests-prefix", default="ultraplot/tests/") + args = parser.parse_args() + + files_map = load_map(args.map_path) + changed_files = read_changed_files(args.changed_files) + + result = {"mode": "full", "tests": []} + if not files_map or not changed_files: + Path(args.output).write_text(json.dumps(result, indent=2), encoding="utf-8") + return 0 + + tests = set() + for path in changed_files: + path = path.replace("\\", "/") + if matches_any(path, args.ignore): + continue + if matches_any(path, args.always_full): + tests.clear() + result["mode"] = "full" + break + if path.startswith(args.tests_prefix): + tests.add(path) + continue + if path in files_map: + tests.update(files_map[path]) + continue + if path.startswith(args.source_prefix): + tests.clear() + result["mode"] = "full" + break + + if tests: + result["mode"] = "selected" + result["tests"] = sorted(tests) + + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(json.dumps(result, indent=2), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())