diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 8a87408d..28c114e5 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -21,9 +21,10 @@ name: Linting and MyPy (Pelican) on: push: paths: - - '**.py' - - '**/linting.yml' - - '**/pylintrc' + - 'pelican/**.py' + - 'pelican/pyproject.toml' + - 'pelican/pylintrc' + - '.github/workflows/linting.yml' workflow_dispatch: permissions: diff --git a/.github/workflows/pelican-action-test.yml b/.github/workflows/pelican-action-test.yml index 42341828..4e5c528d 100644 --- a/.github/workflows/pelican-action-test.yml +++ b/.github/workflows/pelican-action-test.yml @@ -41,7 +41,8 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ env.SOURCE }} - persist-credentials: false + # Credentials must persist: the pelican action's publish step runs + # `git push` against this working tree to commit the built site. - name: Ignore the action checkout run: | echo "self/" >> .git/info/exclude diff --git a/.github/workflows/stash-action-test.yml b/.github/workflows/stash-action-test.yml index 340cd227..aaa04924 100644 --- a/.github/workflows/stash-action-test.yml +++ b/.github/workflows/stash-action-test.yml @@ -71,6 +71,7 @@ jobs: run: | cd stash/restore python3 test_get_stash.py + python3 test_download_stash.py - name: Test Save uses: ./stash/save diff --git a/stash/pyproject.toml b/stash/pyproject.toml new file mode 100644 index 00000000..21b64469 --- /dev/null +++ b/stash/pyproject.toml @@ -0,0 +1,92 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[build-system] +requires = ["setuptools>=82.0.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "apache-stash-action" +version = "0.1.0" +description = "GitHub Action for stashing and restoring build caches via workflow artifacts." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +authors = [ + { name = "Apache Software Foundation", email = "dev@infra.apache.org" }, +] +keywords = ["github-action", "cache", "artifacts", "apache"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Build Tools", +] +# The stash action's Python code is stdlib-only at runtime; the external +# dependencies are the `gh` and `jq` CLIs, which are installed on the +# GitHub Actions runner and checked for at action start-up. +dependencies = [] + +# PEP 735 dependency groups. `dev` holds everything a contributor needs +# to lint and test the stash helpers locally. pytest is listed so the +# existing unittest-based tests can be discovered and run via pytest if +# desired, alongside `python3 test_*.py`. +[dependency-groups] +dev = [ + "ruff>=0.6", + "mypy>=1.10", + "pytest>=8", +] + +[project.urls] +Homepage = "https://github.com/apache/infrastructure-actions" +Source = "https://github.com/apache/infrastructure-actions" +Issues = "https://github.com/apache/infrastructure-actions/issues" + +[tool.setuptools] +py-modules = [] + +[tool.uv] +required-version = ">=0.5.0" + +[tool.hatch.envs.default] +installer = "uv" +dependency-groups = ["dev"] + +[tool.hatch.envs.default.scripts] +lint = "ruff check ." +fmt = "ruff format ." +test = "pytest" + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B", "SIM"] +ignore = ["E501"] + +[tool.ruff.format] +quote-style = "double" diff --git a/stash/restore/action.yml b/stash/restore/action.yml index 50b43f1d..3c88229b 100644 --- a/stash/restore/action.yml +++ b/stash/restore/action.yml @@ -45,6 +45,20 @@ inputs: If true, only the current branch will be searched for the stash. If false, the base branch(PRs)/default branch will be searched as well. default: "false" + retry-count: + description: > + Number of attempts for downloading the stash artifact. When `gh run download` + exits with code 1 (the transient failure mode observed for artifact downloads), + the download will be retried until it succeeds or this many attempts have been + made. Other exit codes are not retried. + default: "3" + fail-on-download: + description: > + If true, the action will fail when a stash artifact was found but could not be + downloaded (after exhausting `retry-count` attempts). If false (the default), + a failed download is reported only via the `stash-hit` output and the step + itself succeeds. + default: "false" outputs: stash-hit: description: > @@ -139,31 +153,23 @@ runs: ) - name: Download Stash - shell: bash + shell: python3 {0} if: steps.check-stash.outputs.stash_found != 'false' id: download env: + PYTHONPATH: "${{ github.action_path }}" GH_TOKEN: "${{ inputs.token }}" STASH_NAME: "${{ steps.check-stash.outputs.stash_name }}" STASH_RUN_ID: "${{ steps.check-stash.outputs.stash_run_id }}" REPO: "${{ github.repository }}" STASH_DIR: "${{ steps.mung.outputs.stash_path }}" - INPUTS_CLEAN: ${{ inputs.clean }} + CLEAN: "${{ inputs.clean }}" + RETRY_COUNT: "${{ inputs.retry-count }}" + FAIL_ON_DOWNLOAD: "${{ inputs.fail-on-download }}" run: | - # Catch errors in the download with || to avoid the whole workflow failing - # when the download times out - if [[ "${INPUTS_CLEAN}" == "true" ]]; then - if [[ -d "$STASH_DIR" ]]; then - echo "Removing existing stash directory: $STASH_DIR" - rm -rf "$STASH_DIR" - fi - fi - gh run download "$STASH_RUN_ID" \ - --name "$STASH_NAME" \ - --dir "$STASH_DIR" \ - -R "$REPO" || download="failed" && download="success" - - echo "download=$download" >> "$GITHUB_OUTPUT" + import os, sys + import download_stash + sys.exit(download_stash.download_stash(os.environ)) - name: Set stash-hit Output id: output diff --git a/stash/restore/download_stash.py b/stash/restore/download_stash.py new file mode 100644 index 00000000..5e14818a --- /dev/null +++ b/stash/restore/download_stash.py @@ -0,0 +1,102 @@ +# Copyright (c) The stash contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Download a stash artifact with retries. + +Required env vars: + STASH_RUN_ID - workflow run ID the artifact was produced by + STASH_NAME - artifact name + STASH_DIR - destination directory + REPO - owner/name of the repository + RETRY_COUNT - max download attempts (retries on gh exit code 1) + FAIL_ON_DOWNLOAD - "true" to exit 1 on download failure, else "false" + CLEAN - "true" to remove STASH_DIR before downloading + GITHUB_OUTPUT - file to write the `download` output to +""" + +import os +import shutil +import subprocess +import sys +from collections.abc import Callable, Mapping + + +def run_gh_download(run_id: str, name: str, dest: str, repo: str) -> int: + """Invoke ``gh run download`` and return its exit code.""" + return subprocess.run( + [ + "gh", "run", "download", run_id, + "--name", name, + "--dir", dest, + "-R", repo, + ], + check=False, + ).returncode + + +def download_stash( + env: Mapping[str, str], + run_download: Callable[[str, str, str, str], int] = run_gh_download, +) -> int: + """Run the clean/retry/fail-on-download logic. + + Returns the desired process exit code (0 for success or tolerated + failure, 1 when the download failed and ``FAIL_ON_DOWNLOAD`` is + ``"true"``). The ``run_download`` hook exists so tests can stub out + the real ``gh`` call. + """ + stash_run_id = env["STASH_RUN_ID"] + stash_name = env["STASH_NAME"] + stash_dir = env["STASH_DIR"] + repo = env["REPO"] + retry_count = int(env.get("RETRY_COUNT", "1")) + fail_on_download = env.get("FAIL_ON_DOWNLOAD", "false").lower() == "true" + clean = env.get("CLEAN", "false").lower() == "true" + github_output = env["GITHUB_OUTPUT"] + + if clean and os.path.isdir(stash_dir): + print(f"Removing existing stash directory: {stash_dir}") + shutil.rmtree(stash_dir, ignore_errors=True) + + download = "failed" + for attempt in range(1, retry_count + 1): + print(f"Downloading stash (attempt {attempt} of {retry_count})...", flush=True) + rc = run_download(stash_run_id, stash_name, stash_dir, repo) + if rc == 0: + download = "success" + break + if rc != 1: + print( + f"::warning ::gh run download failed with exit code {rc}; " + "not retrying." + ) + break + print( + f"::warning ::gh run download failed with exit code 1 on " + f"attempt {attempt}." + ) + + with open(github_output, "a", encoding="utf-8") as f: + f.write(f"download={download}\n") + + if download != "success" and fail_on_download: + print( + f"::error ::Stash artifact download failed after {retry_count} " + "attempt(s) and fail-on-download is true." + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(download_stash(os.environ)) diff --git a/stash/restore/test_download_stash.py b/stash/restore/test_download_stash.py new file mode 100644 index 00000000..5b31ea06 --- /dev/null +++ b/stash/restore/test_download_stash.py @@ -0,0 +1,136 @@ +# Copyright (c) The stash contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for download_stash.download_stash.""" + +import tempfile +import unittest +from pathlib import Path + +from download_stash import download_stash + + +class FakeGh: + """Emits successive exit codes for ``gh run download`` invocations. + + After the provided list of codes is exhausted, the last code is + reused for any further calls. Each call is recorded so tests can + assert how many times ``gh`` was invoked. + """ + + def __init__(self, codes): + self.codes = list(codes) + self.calls = [] + + def __call__(self, run_id, name, dest, repo): + self.calls.append((run_id, name, dest, repo)) + idx = min(len(self.calls) - 1, len(self.codes) - 1) + return self.codes[idx] + + +class TestDownloadStash(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.stash_dir = self.tmp / "target" + self.stash_dir.mkdir() + self.output_file = self.tmp / "github_output" + self.output_file.touch() + + def tearDown(self): + self._tmp.cleanup() + + def env(self, **overrides): + base = { + "STASH_RUN_ID": "42", + "STASH_NAME": "fake-stash", + "STASH_DIR": str(self.stash_dir), + "REPO": "test/repo", + "RETRY_COUNT": "3", + "FAIL_ON_DOWNLOAD": "false", + "CLEAN": "false", + "GITHUB_OUTPUT": str(self.output_file), + } + base.update(overrides) + return base + + def read_output(self): + return self.output_file.read_text() + + def test_success_first_attempt(self): + gh = FakeGh([0]) + rc = download_stash(self.env(), run_download=gh) + self.assertEqual(rc, 0) + self.assertIn("download=success", self.read_output()) + self.assertEqual(len(gh.calls), 1) + + def test_retry_on_exit_1_until_success(self): + gh = FakeGh([1, 1, 0]) + rc = download_stash(self.env(), run_download=gh) + self.assertEqual(rc, 0) + self.assertIn("download=success", self.read_output()) + self.assertEqual(len(gh.calls), 3) + + def test_all_retries_fail_tolerated(self): + gh = FakeGh([1]) + rc = download_stash(self.env(), run_download=gh) + self.assertEqual(rc, 0) + self.assertIn("download=failed", self.read_output()) + self.assertEqual(len(gh.calls), 3) + + def test_all_retries_fail_fail_on_download(self): + gh = FakeGh([1]) + rc = download_stash( + self.env(RETRY_COUNT="2", FAIL_ON_DOWNLOAD="true"), + run_download=gh, + ) + self.assertEqual(rc, 1) + self.assertIn("download=failed", self.read_output()) + self.assertEqual(len(gh.calls), 2) + + def test_non_transient_exit_not_retried(self): + gh = FakeGh([2]) + rc = download_stash(self.env(RETRY_COUNT="5"), run_download=gh) + self.assertEqual(rc, 0) + self.assertIn("download=failed", self.read_output()) + self.assertEqual(len(gh.calls), 1) + + def test_clean_removes_stash_dir(self): + (self.stash_dir / "leftover").touch() + gh = FakeGh([0]) + rc = download_stash( + self.env(CLEAN="true", RETRY_COUNT="1"), run_download=gh + ) + self.assertEqual(rc, 0) + self.assertFalse((self.stash_dir / "leftover").exists()) + self.assertIn("download=success", self.read_output()) + + def test_clean_false_preserves_stash_dir(self): + (self.stash_dir / "leftover").touch() + gh = FakeGh([0]) + rc = download_stash( + self.env(CLEAN="false", RETRY_COUNT="1"), run_download=gh + ) + self.assertEqual(rc, 0) + self.assertTrue((self.stash_dir / "leftover").exists()) + + def test_stops_on_first_non_transient(self): + gh = FakeGh([1, 2, 0]) + rc = download_stash(self.env(RETRY_COUNT="5"), run_download=gh) + self.assertEqual(rc, 0) + self.assertIn("download=failed", self.read_output()) + self.assertEqual(len(gh.calls), 2) + + +if __name__ == "__main__": + unittest.main()