From 4b8bddb7609ff45e510737f40f6fc1aeb120ecf9 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 13:09:57 -0700 Subject: [PATCH 01/24] ci: add GitLab pipeline to bump rshell in datadog-agent on new tag --- .gitlab-ci.yml | 24 +++++ .gitlab/scripts/bump_datadog_agent.py | 141 ++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100755 .gitlab/scripts/bump_datadog_agent.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..2e4960e1 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,24 @@ +--- +stages: + - trigger_release + +.dd_octo_sts: + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + +.setup_github_bump_token: + - export GITHUB_TOKEN=$(dd-octo-sts token --scope DataDog/datadog-agent --policy self.rshell.bump-rshell-version) + +bump_datadog_agent: + stage: trigger_release + image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux:latest + tags: ["arch:arm64"] + extends: .dd_octo_sts + resource_group: rshell-bump + rules: + - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ + before_script: + - !reference [.setup_github_bump_token] + script: + - python3 .gitlab/scripts/bump_datadog_agent.py "$CI_COMMIT_TAG" diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py new file mode 100755 index 00000000..b12ec965 --- /dev/null +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Open a PR on DataDog/datadog-agent that bumps the pinned rshell version. + +Invoked by the `bump_datadog_agent` GitLab CI job after a new rshell tag is +detected. Expects: + - sys.argv[1]: the rshell tag (e.g. "v0.0.11") + - env GITHUB_TOKEN: a short-lived dd-octo-sts token scoped to + DataDog/datadog-agent with contents:write + pull-requests:write. +""" + +from __future__ import annotations + +import os +import re +import secrets +import subprocess +import sys +from pathlib import Path + +from github import Auth, Github, GithubException + +TARGET_REPO = "DataDog/datadog-agent" +TARGET_BASE = "main" +RSHELL_MODULE = "github.com/DataDog/rshell" +REVIEW_TEAM = "action-platform" +PR_LABELS = ["changelog/no-changelog", "ask-review"] +GIT_USER_NAME = "github-actions[bot]" +GIT_USER_EMAIL = "github-actions[bot]@users.noreply.github.com" + + +def run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: + print(f"+ {' '.join(cmd)}", flush=True) + return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) + + +def strip_rshell_replace(go_mod: Path) -> None: + """Drop any `replace github.com/DataDog/rshell => ...` line (one-time v0.0.11 transition).""" + original = go_mod.read_text() + pattern = re.compile(rf"^\s*replace\s+{re.escape(RSHELL_MODULE)}\s+=>.*$\n?", re.MULTILINE) + updated = pattern.sub("", original) + if updated != original: + go_mod.write_text(updated) + print(f"stripped replace directive for {RSHELL_MODULE}", flush=True) + + +def current_rshell_version(go_mod: Path) -> str | None: + pattern = re.compile(rf"^\s*{re.escape(RSHELL_MODULE)}\s+(v\S+)", re.MULTILINE) + m = pattern.search(go_mod.read_text()) + return m.group(1) if m else None + + +def write_release_note(repo_root: Path, version: str) -> Path: + suffix = secrets.token_hex(8) + note = repo_root / "releasenotes" / "notes" / f"bump-rshell-{version}-{suffix}.yaml" + note.write_text( + "---\n" + "enhancements:\n" + " - |\n" + f" Bump ``rshell`` to {version} for the Private Action Runner.\n" + ) + return note + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: bump_datadog_agent.py ", file=sys.stderr) + return 2 + version = sys.argv[1] + if not re.fullmatch(r"v\d+\.\d+\.\d+", version): + print(f"invalid version {version!r}; expected vX.Y.Z", file=sys.stderr) + return 2 + + token = os.environ.get("GITHUB_TOKEN") + if not token: + print("GITHUB_TOKEN is not set; dd-octo-sts exchange failed upstream", file=sys.stderr) + return 1 + + gh = Github(auth=Auth.Token(token), per_page=100) + repo = gh.get_repo(TARGET_REPO) + branch = f"bot/bump-rshell-{version}" + + existing = list(repo.get_pulls(state="open", head=f"{TARGET_REPO.split('/')[0]}:{branch}")) + if existing: + print(f"PR already exists for {branch}: {existing[0].html_url}; nothing to do", flush=True) + return 0 + + workdir = Path("/tmp/datadog-agent") + clone_url = f"https://x-access-token:{token}@github.com/{TARGET_REPO}.git" + run(["git", "clone", "--depth=1", "--branch", TARGET_BASE, clone_url, str(workdir)]) + run(["git", "config", "user.name", GIT_USER_NAME], cwd=workdir) + run(["git", "config", "user.email", GIT_USER_EMAIL], cwd=workdir) + run(["git", "checkout", "-b", branch], cwd=workdir) + + go_mod = workdir / "go.mod" + previous_version = current_rshell_version(go_mod) + strip_rshell_replace(go_mod) + run(["go", "get", f"{RSHELL_MODULE}@{version}"], cwd=workdir) + run(["dda", "inv", "tidy"], cwd=workdir) + write_release_note(workdir, version) + + run(["git", "add", "-A"], cwd=workdir) + diff = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=workdir) + if diff.returncode == 0: + print(f"no changes to commit; datadog-agent already at rshell {version}", flush=True) + return 0 + + commit_msg = ( + f"Bump rshell dependency from {previous_version} to {version}" + if previous_version + else f"Bump rshell dependency to {version}" + ) + run(["git", "commit", "-m", commit_msg], cwd=workdir) + run(["git", "push", "origin", branch], cwd=workdir) + + pr = repo.create_pull( + title=f"[automated] Bump rshell to {version}", + body=( + f"Automated bump of `{RSHELL_MODULE}` to " + f"[{version}](https://github.com/DataDog/rshell/releases/tag/{version}).\n" + ), + base=TARGET_BASE, + head=branch, + draft=True, + ) + print(f"opened draft PR: {pr.html_url}", flush=True) + + try: + pr.add_to_labels(*PR_LABELS) + except GithubException as e: + print(f"warning: failed to add labels {PR_LABELS}: {e}", flush=True) + + try: + pr.create_review_request(team_reviewers=[REVIEW_TEAM]) + except GithubException as e: + print(f"warning: failed to request review from @DataDog/{REVIEW_TEAM}: {e}", flush=True) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 513701b0134821464e984dd534d998e74f1ab893 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 14:36:06 -0700 Subject: [PATCH 02/24] ci: allow manual web trigger with BUMP_VERSION variable --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e4960e1..32138b2d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,8 @@ bump_datadog_agent: resource_group: rshell-bump rules: - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ + - if: $CI_PIPELINE_SOURCE == "web" && $BUMP_VERSION =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ before_script: - !reference [.setup_github_bump_token] script: - - python3 .gitlab/scripts/bump_datadog_agent.py "$CI_COMMIT_TAG" + - python3 .gitlab/scripts/bump_datadog_agent.py "${CI_COMMIT_TAG:-$BUMP_VERSION}" From b1ad5977d3c9de3894e13e52544a34f2e08f318f Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 14:47:43 -0700 Subject: [PATCH 03/24] ci(bump): drop bot/ prefix on bump branch --- .gitlab/scripts/bump_datadog_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index b12ec965..d88f9e34 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -77,7 +77,7 @@ def main() -> int: gh = Github(auth=Auth.Token(token), per_page=100) repo = gh.get_repo(TARGET_REPO) - branch = f"bot/bump-rshell-{version}" + branch = f"bump-rshell-{version}" existing = list(repo.get_pulls(state="open", head=f"{TARGET_REPO.split('/')[0]}:{branch}")) if existing: From 2acb6bfd9c83005d253ab2710dd5d696434c5dab Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 14:52:58 -0700 Subject: [PATCH 04/24] ci(bump): clone datadog-agent anonymously; only auth on push --- .gitlab/scripts/bump_datadog_agent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index d88f9e34..d5a0bc3a 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -85,7 +85,7 @@ def main() -> int: return 0 workdir = Path("/tmp/datadog-agent") - clone_url = f"https://x-access-token:{token}@github.com/{TARGET_REPO}.git" + clone_url = f"https://github.com/{TARGET_REPO}.git" run(["git", "clone", "--depth=1", "--branch", TARGET_BASE, clone_url, str(workdir)]) run(["git", "config", "user.name", GIT_USER_NAME], cwd=workdir) run(["git", "config", "user.email", GIT_USER_EMAIL], cwd=workdir) @@ -110,7 +110,8 @@ def main() -> int: else f"Bump rshell dependency to {version}" ) run(["git", "commit", "-m", commit_msg], cwd=workdir) - run(["git", "push", "origin", branch], cwd=workdir) + push_url = f"https://x-access-token:{token}@github.com/{TARGET_REPO}.git" + run(["git", "push", push_url, branch], cwd=workdir) pr = repo.create_pull( title=f"[automated] Bump rshell to {version}", From e7ee0eb3ec186f8ee56f0d6ce93359e442060c53 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 14:58:38 -0700 Subject: [PATCH 05/24] test(bump): add stdlib unit tests for bump_datadog_agent.py --- .gitignore | 2 + .gitlab/scripts/bump_datadog_agent.py | 10 +- .gitlab/scripts/test_bump_datadog_agent.py | 192 +++++++++++++++++++++ 3 files changed, 201 insertions(+), 3 deletions(-) create mode 100644 .gitlab/scripts/test_bump_datadog_agent.py diff --git a/.gitignore b/.gitignore index fe006405..d58c8e31 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ .DS_Store +__pycache__/ + # Fuzz corpus: keep checked in for regression testing. # Uncomment the line below if corpus grows too large: # interp/builtins/tests/*/testdata/fuzz/*/corpus-* diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index d5a0bc3a..d5d71ad2 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -17,8 +17,6 @@ import sys from pathlib import Path -from github import Auth, Github, GithubException - TARGET_REPO = "DataDog/datadog-agent" TARGET_BASE = "main" RSHELL_MODULE = "github.com/DataDog/rshell" @@ -44,7 +42,11 @@ def strip_rshell_replace(go_mod: Path) -> None: def current_rshell_version(go_mod: Path) -> str | None: - pattern = re.compile(rf"^\s*{re.escape(RSHELL_MODULE)}\s+(v\S+)", re.MULTILINE) + """Return the rshell version pinned in a `require` declaration, ignoring `replace` lines.""" + pattern = re.compile( + rf"^\s*(?:require\s+)?{re.escape(RSHELL_MODULE)}\s+(v\S+)(?!\s*=>)", + re.MULTILINE, + ) m = pattern.search(go_mod.read_text()) return m.group(1) if m else None @@ -75,6 +77,8 @@ def main() -> int: print("GITHUB_TOKEN is not set; dd-octo-sts exchange failed upstream", file=sys.stderr) return 1 + from github import Auth, Github, GithubException + gh = Github(auth=Auth.Token(token), per_page=100) repo = gh.get_repo(TARGET_REPO) branch = f"bump-rshell-{version}" diff --git a/.gitlab/scripts/test_bump_datadog_agent.py b/.gitlab/scripts/test_bump_datadog_agent.py new file mode 100644 index 00000000..f4890021 --- /dev/null +++ b/.gitlab/scripts/test_bump_datadog_agent.py @@ -0,0 +1,192 @@ +"""Tests for bump_datadog_agent.py. + +Runs with stdlib only; PyGithub is stubbed before the script is imported, so +the suite executes anywhere Python 3.10+ is installed: + + python3 -m unittest .gitlab/scripts/test_bump_datadog_agent.py +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +_github_stub = MagicMock() +_github_stub.GithubException = type("GithubException", (Exception,), {}) +sys.modules["github"] = _github_stub + +sys.path.insert(0, str(Path(__file__).parent)) +import bump_datadog_agent as bump # noqa: E402 + + +class TestStripRshellReplace(unittest.TestCase): + def test_removes_replace_directive(self): + original = textwrap.dedent( + """\ + module m + go 1.25 + + replace github.com/DataDog/rshell => /local/path + + require ( + github.com/DataDog/rshell v0.0.10 + ) + """ + ) + with tempfile.TemporaryDirectory() as td: + go_mod = Path(td) / "go.mod" + go_mod.write_text(original) + bump.strip_rshell_replace(go_mod) + updated = go_mod.read_text() + self.assertNotIn("replace github.com/DataDog/rshell", updated) + self.assertIn("require (", updated) + self.assertIn("github.com/DataDog/rshell v0.0.10", updated) + + def test_noop_when_no_replace(self): + original = "module m\ngo 1.25\n\nrequire github.com/DataDog/rshell v0.0.10\n" + with tempfile.TemporaryDirectory() as td: + go_mod = Path(td) / "go.mod" + go_mod.write_text(original) + bump.strip_rshell_replace(go_mod) + self.assertEqual(go_mod.read_text(), original) + + def test_preserves_other_replaces(self): + original = textwrap.dedent( + """\ + module m + go 1.25 + + replace github.com/other/mod => /a + replace github.com/DataDog/rshell => /local/path + replace github.com/yet/another => /b + """ + ) + with tempfile.TemporaryDirectory() as td: + go_mod = Path(td) / "go.mod" + go_mod.write_text(original) + bump.strip_rshell_replace(go_mod) + updated = go_mod.read_text() + self.assertNotIn("DataDog/rshell", updated) + self.assertIn("github.com/other/mod", updated) + self.assertIn("github.com/yet/another", updated) + + +class TestCurrentRshellVersion(unittest.TestCase): + def _write(self, content: str) -> Path: + td = tempfile.TemporaryDirectory() + self.addCleanup(td.cleanup) + go_mod = Path(td.name) / "go.mod" + go_mod.write_text(content) + return go_mod + + def test_finds_version_in_require_block(self): + go_mod = self._write( + textwrap.dedent( + """\ + require ( + github.com/DataDog/rshell v0.0.10 + github.com/DataDog/other v1.2.3 + ) + """ + ) + ) + self.assertEqual(bump.current_rshell_version(go_mod), "v0.0.10") + + def test_finds_version_in_single_require(self): + go_mod = self._write("require github.com/DataDog/rshell v0.0.11\n") + self.assertEqual(bump.current_rshell_version(go_mod), "v0.0.11") + + def test_ignores_replace_line_version(self): + go_mod = self._write( + textwrap.dedent( + """\ + require github.com/DataDog/rshell v0.0.10 + replace github.com/DataDog/rshell v0.0.10 => /local + """ + ) + ) + self.assertEqual(bump.current_rshell_version(go_mod), "v0.0.10") + + def test_returns_none_when_absent(self): + go_mod = self._write("module m\ngo 1.25\n") + self.assertIsNone(bump.current_rshell_version(go_mod)) + + +class TestWriteReleaseNote(unittest.TestCase): + def test_writes_correct_format_and_path(self): + with tempfile.TemporaryDirectory() as td: + repo_root = Path(td) + (repo_root / "releasenotes" / "notes").mkdir(parents=True) + note = bump.write_release_note(repo_root, "v0.0.11") + self.assertTrue(note.exists()) + self.assertEqual(note.parent, repo_root / "releasenotes" / "notes") + self.assertTrue(note.name.startswith("bump-rshell-v0.0.11-")) + self.assertTrue(note.name.endswith(".yaml")) + content = note.read_text() + self.assertIn("enhancements:", content) + self.assertIn("Bump ``rshell`` to v0.0.11", content) + + def test_filenames_are_unique(self): + with tempfile.TemporaryDirectory() as td: + repo_root = Path(td) + (repo_root / "releasenotes" / "notes").mkdir(parents=True) + a = bump.write_release_note(repo_root, "v0.0.11") + b = bump.write_release_note(repo_root, "v0.0.11") + self.assertNotEqual(a.name, b.name) + + +class TestMainInputValidation(unittest.TestCase): + def test_missing_argv_returns_2(self): + with patch.object(sys, "argv", ["bump_datadog_agent.py"]): + self.assertEqual(bump.main(), 2) + + def test_extra_argv_returns_2(self): + with patch.object(sys, "argv", ["bump_datadog_agent.py", "v0.0.11", "extra"]): + self.assertEqual(bump.main(), 2) + + def test_rejects_version_without_v_prefix(self): + with patch.object(sys, "argv", ["bump_datadog_agent.py", "0.0.11"]): + self.assertEqual(bump.main(), 2) + + def test_rejects_non_semver(self): + for bad in ("v0.0", "v1.2.3-alpha", "vX.Y.Z", "v1.2.3.4"): + with self.subTest(bad=bad): + with patch.object(sys, "argv", ["bump_datadog_agent.py", bad]): + self.assertEqual(bump.main(), 2) + + def test_missing_github_token_returns_1(self): + with patch.dict(os.environ, {}, clear=True): + with patch.object(sys, "argv", ["bump_datadog_agent.py", "v0.0.11"]): + self.assertEqual(bump.main(), 1) + + +class TestMainIdempotency(unittest.TestCase): + def test_exits_zero_when_pr_already_exists(self): + existing_pr = MagicMock() + existing_pr.html_url = "https://github.com/DataDog/datadog-agent/pull/999" + mock_repo = MagicMock() + mock_repo.get_pulls.return_value = [existing_pr] + _github_stub.Github.reset_mock() + _github_stub.Github.return_value.get_repo.return_value = mock_repo + + with patch.dict(os.environ, {"GITHUB_TOKEN": "fake-token"}): + with patch.object(sys, "argv", ["bump_datadog_agent.py", "v0.0.99"]): + # subprocess.run should never be reached on the early-exit path + with patch("bump_datadog_agent.subprocess.run") as mock_run: + result = bump.main() + mock_run.assert_not_called() + + self.assertEqual(result, 0) + mock_repo.get_pulls.assert_called_once() + call_kwargs = mock_repo.get_pulls.call_args.kwargs + self.assertEqual(call_kwargs["state"], "open") + self.assertEqual(call_kwargs["head"], "DataDog:bump-rshell-v0.0.99") + + +if __name__ == "__main__": + unittest.main() From a5d01812425a5d428b3d07b98206aa53b3b54536 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 15:32:27 -0700 Subject: [PATCH 06/24] ci(bump): redact credentials in logged command lines --- .gitlab/scripts/bump_datadog_agent.py | 10 ++++++- .gitlab/scripts/test_bump_datadog_agent.py | 31 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index d5d71ad2..52101e27 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -26,8 +26,16 @@ GIT_USER_EMAIL = "github-actions[bot]@users.noreply.github.com" +_CREDS_IN_URL = re.compile(r"(https?://[^/@:\s]+):[^@/\s]+@") + + +def _redact(cmd: list[str]) -> list[str]: + """Redact credentials embedded in URL-shaped arguments for safe logging.""" + return [_CREDS_IN_URL.sub(r"\1:@", arg) for arg in cmd] + + def run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: - print(f"+ {' '.join(cmd)}", flush=True) + print(f"+ {' '.join(_redact(cmd))}", flush=True) return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) diff --git a/.gitlab/scripts/test_bump_datadog_agent.py b/.gitlab/scripts/test_bump_datadog_agent.py index f4890021..139a9b68 100644 --- a/.gitlab/scripts/test_bump_datadog_agent.py +++ b/.gitlab/scripts/test_bump_datadog_agent.py @@ -24,6 +24,37 @@ import bump_datadog_agent as bump # noqa: E402 +class TestRedact(unittest.TestCase): + def test_redacts_token_in_push_url(self): + cmd = ["git", "push", "https://x-access-token:ghs_SECRET123@github.com/DataDog/datadog-agent.git", "main"] + redacted = bump._redact(cmd) + self.assertNotIn("ghs_SECRET123", " ".join(redacted)) + self.assertIn("", redacted[2]) + self.assertIn("x-access-token", redacted[2]) + self.assertIn("github.com/DataDog/datadog-agent.git", redacted[2]) + + def test_leaves_clean_urls_alone(self): + cmd = ["git", "clone", "https://github.com/DataDog/datadog-agent.git", "/tmp/x"] + self.assertEqual(bump._redact(cmd), cmd) + + def test_leaves_non_url_args_alone(self): + cmd = ["git", "config", "user.email", "bot@users.noreply.github.com"] + self.assertEqual(bump._redact(cmd), cmd) + + def test_handles_multiple_tokened_args(self): + cmd = [ + "echo", + "https://user1:tok1@host.a/repo", + "plain-arg", + "https://user2:tok2@host.b/repo", + ] + redacted = bump._redact(cmd) + joined = " ".join(redacted) + self.assertNotIn("tok1", joined) + self.assertNotIn("tok2", joined) + self.assertEqual(redacted.count("plain-arg"), 1) + + class TestStripRshellReplace(unittest.TestCase): def test_removes_replace_directive(self): original = textwrap.dedent( From d63092d16c10d073aa40ca5999163b064b8278df Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 15:35:18 -0700 Subject: [PATCH 07/24] ci(bump): stop logging commands; move token to git credentials file --- .gitlab/scripts/bump_datadog_agent.py | 37 ++++++++-------- .gitlab/scripts/test_bump_datadog_agent.py | 51 ++++++++++------------ 2 files changed, 41 insertions(+), 47 deletions(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index 52101e27..deca1321 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -26,17 +26,22 @@ GIT_USER_EMAIL = "github-actions[bot]@users.noreply.github.com" -_CREDS_IN_URL = re.compile(r"(https?://[^/@:\s]+):[^@/\s]+@") - +def run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) -def _redact(cmd: list[str]) -> list[str]: - """Redact credentials embedded in URL-shaped arguments for safe logging.""" - return [_CREDS_IN_URL.sub(r"\1:@", arg) for arg in cmd] +def configure_credentials(workdir: Path, token: str) -> Path: + """Store the GitHub token in a local git credentials file under .git/. -def run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: - print(f"+ {' '.join(_redact(cmd))}", flush=True) - return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) + Keeps the token out of process argv (visible to `ps`), command-line logs, + and subprocess exception tracebacks. The file is mode 0600 and lives inside + the ephemeral clone directory, which is discarded when the runner exits. + """ + creds_path = workdir / ".git" / "ci-credentials" + creds_path.write_text(f"https://x-access-token:{token}@github.com\n") + creds_path.chmod(0o600) + run(["git", "config", "credential.helper", f"store --file={creds_path}"], cwd=workdir) + return creds_path def strip_rshell_replace(go_mod: Path) -> None: @@ -46,7 +51,6 @@ def strip_rshell_replace(go_mod: Path) -> None: updated = pattern.sub("", original) if updated != original: go_mod.write_text(updated) - print(f"stripped replace directive for {RSHELL_MODULE}", flush=True) def current_rshell_version(go_mod: Path) -> str | None: @@ -93,12 +97,12 @@ def main() -> int: existing = list(repo.get_pulls(state="open", head=f"{TARGET_REPO.split('/')[0]}:{branch}")) if existing: - print(f"PR already exists for {branch}: {existing[0].html_url}; nothing to do", flush=True) return 0 workdir = Path("/tmp/datadog-agent") clone_url = f"https://github.com/{TARGET_REPO}.git" run(["git", "clone", "--depth=1", "--branch", TARGET_BASE, clone_url, str(workdir)]) + configure_credentials(workdir, token) run(["git", "config", "user.name", GIT_USER_NAME], cwd=workdir) run(["git", "config", "user.email", GIT_USER_EMAIL], cwd=workdir) run(["git", "checkout", "-b", branch], cwd=workdir) @@ -113,7 +117,6 @@ def main() -> int: run(["git", "add", "-A"], cwd=workdir) diff = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=workdir) if diff.returncode == 0: - print(f"no changes to commit; datadog-agent already at rshell {version}", flush=True) return 0 commit_msg = ( @@ -122,8 +125,7 @@ def main() -> int: else f"Bump rshell dependency to {version}" ) run(["git", "commit", "-m", commit_msg], cwd=workdir) - push_url = f"https://x-access-token:{token}@github.com/{TARGET_REPO}.git" - run(["git", "push", push_url, branch], cwd=workdir) + run(["git", "push", "origin", branch], cwd=workdir) pr = repo.create_pull( title=f"[automated] Bump rshell to {version}", @@ -135,17 +137,16 @@ def main() -> int: head=branch, draft=True, ) - print(f"opened draft PR: {pr.html_url}", flush=True) try: pr.add_to_labels(*PR_LABELS) - except GithubException as e: - print(f"warning: failed to add labels {PR_LABELS}: {e}", flush=True) + except GithubException: + pass try: pr.create_review_request(team_reviewers=[REVIEW_TEAM]) - except GithubException as e: - print(f"warning: failed to request review from @DataDog/{REVIEW_TEAM}: {e}", flush=True) + except GithubException: + pass return 0 diff --git a/.gitlab/scripts/test_bump_datadog_agent.py b/.gitlab/scripts/test_bump_datadog_agent.py index 139a9b68..c8571cba 100644 --- a/.gitlab/scripts/test_bump_datadog_agent.py +++ b/.gitlab/scripts/test_bump_datadog_agent.py @@ -24,35 +24,28 @@ import bump_datadog_agent as bump # noqa: E402 -class TestRedact(unittest.TestCase): - def test_redacts_token_in_push_url(self): - cmd = ["git", "push", "https://x-access-token:ghs_SECRET123@github.com/DataDog/datadog-agent.git", "main"] - redacted = bump._redact(cmd) - self.assertNotIn("ghs_SECRET123", " ".join(redacted)) - self.assertIn("", redacted[2]) - self.assertIn("x-access-token", redacted[2]) - self.assertIn("github.com/DataDog/datadog-agent.git", redacted[2]) - - def test_leaves_clean_urls_alone(self): - cmd = ["git", "clone", "https://github.com/DataDog/datadog-agent.git", "/tmp/x"] - self.assertEqual(bump._redact(cmd), cmd) - - def test_leaves_non_url_args_alone(self): - cmd = ["git", "config", "user.email", "bot@users.noreply.github.com"] - self.assertEqual(bump._redact(cmd), cmd) - - def test_handles_multiple_tokened_args(self): - cmd = [ - "echo", - "https://user1:tok1@host.a/repo", - "plain-arg", - "https://user2:tok2@host.b/repo", - ] - redacted = bump._redact(cmd) - joined = " ".join(redacted) - self.assertNotIn("tok1", joined) - self.assertNotIn("tok2", joined) - self.assertEqual(redacted.count("plain-arg"), 1) +class TestConfigureCredentials(unittest.TestCase): + def test_writes_credentials_file_and_invokes_git_config(self): + with tempfile.TemporaryDirectory() as td: + workdir = Path(td) + (workdir / ".git").mkdir() + with patch("bump_datadog_agent.subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + path = bump.configure_credentials(workdir, "ghs_SECRET123") + + self.assertEqual(path, workdir / ".git" / "ci-credentials") + self.assertTrue(path.exists()) + self.assertIn("ghs_SECRET123", path.read_text()) + self.assertIn("x-access-token", path.read_text()) + # file permissions are 0o600 + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + + # git config was invoked with the path, not the token + self.assertEqual(mock_run.call_count, 1) + called_cmd = mock_run.call_args.args[0] + self.assertEqual(called_cmd[:3], ["git", "config", "credential.helper"]) + self.assertNotIn("ghs_SECRET123", " ".join(called_cmd)) + self.assertIn(str(path), called_cmd[3]) class TestStripRshellReplace(unittest.TestCase): From 26f0d9831a791efab3c0434cb8dd260efb0bebb5 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 15:38:30 -0700 Subject: [PATCH 08/24] ci(bump): add progress logging that never touches credentials --- .gitlab/scripts/bump_datadog_agent.py | 33 +++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index deca1321..b174e6f2 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -30,6 +30,11 @@ def run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subproce return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) +def log(msg: str) -> None: + """Emit a progress line to stdout. Call sites must never pass secrets.""" + print(f"[bump] {msg}", flush=True) + + def configure_credentials(workdir: Path, token: str) -> Path: """Store the GitHub token in a local git credentials file under .git/. @@ -51,6 +56,7 @@ def strip_rshell_replace(go_mod: Path) -> None: updated = pattern.sub("", original) if updated != original: go_mod.write_text(updated) + log(f"stripped `replace {RSHELL_MODULE} =>` directive from go.mod") def current_rshell_version(go_mod: Path) -> str | None: @@ -89,34 +95,45 @@ def main() -> int: print("GITHUB_TOKEN is not set; dd-octo-sts exchange failed upstream", file=sys.stderr) return 1 + log(f"preparing bump of {RSHELL_MODULE} to {version}") from github import Auth, Github, GithubException gh = Github(auth=Auth.Token(token), per_page=100) repo = gh.get_repo(TARGET_REPO) branch = f"bump-rshell-{version}" + log(f"checking {TARGET_REPO} for existing PR with head={branch}") existing = list(repo.get_pulls(state="open", head=f"{TARGET_REPO.split('/')[0]}:{branch}")) if existing: + log(f"PR already exists: {existing[0].html_url}; nothing to do") return 0 workdir = Path("/tmp/datadog-agent") clone_url = f"https://github.com/{TARGET_REPO}.git" + log(f"cloning {TARGET_REPO}@{TARGET_BASE} into {workdir}") run(["git", "clone", "--depth=1", "--branch", TARGET_BASE, clone_url, str(workdir)]) + log("configuring git credentials (token stored in .git/ci-credentials, not argv)") configure_credentials(workdir, token) run(["git", "config", "user.name", GIT_USER_NAME], cwd=workdir) run(["git", "config", "user.email", GIT_USER_EMAIL], cwd=workdir) + log(f"creating branch {branch}") run(["git", "checkout", "-b", branch], cwd=workdir) go_mod = workdir / "go.mod" previous_version = current_rshell_version(go_mod) + log(f"current pinned version in go.mod: {previous_version or ''}") strip_rshell_replace(go_mod) + log(f"running: go get {RSHELL_MODULE}@{version}") run(["go", "get", f"{RSHELL_MODULE}@{version}"], cwd=workdir) + log("running: dda inv tidy") run(["dda", "inv", "tidy"], cwd=workdir) - write_release_note(workdir, version) + note = write_release_note(workdir, version) + log(f"wrote release note: {note.relative_to(workdir)}") run(["git", "add", "-A"], cwd=workdir) diff = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=workdir) if diff.returncode == 0: + log(f"no staged changes; datadog-agent already at rshell {version}") return 0 commit_msg = ( @@ -124,9 +141,12 @@ def main() -> int: if previous_version else f"Bump rshell dependency to {version}" ) + log(f"committing: {commit_msg}") run(["git", "commit", "-m", commit_msg], cwd=workdir) + log(f"pushing branch {branch} to origin") run(["git", "push", "origin", branch], cwd=workdir) + log("opening draft PR") pr = repo.create_pull( title=f"[automated] Bump rshell to {version}", body=( @@ -137,16 +157,19 @@ def main() -> int: head=branch, draft=True, ) + log(f"opened draft PR: {pr.html_url}") try: pr.add_to_labels(*PR_LABELS) - except GithubException: - pass + log(f"added labels: {', '.join(PR_LABELS)}") + except GithubException as e: + log(f"warning: failed to add labels {PR_LABELS}: {e}") try: pr.create_review_request(team_reviewers=[REVIEW_TEAM]) - except GithubException: - pass + log(f"requested review from @DataDog/{REVIEW_TEAM}") + except GithubException as e: + log(f"warning: failed to request review from @DataDog/{REVIEW_TEAM}: {e}") return 0 From 9e0ed9f8cf2fdf8be13a799b04dddd232c20abc2 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 15:42:33 -0700 Subject: [PATCH 09/24] ci(bump): drop PyGithub; use stdlib urllib for GitHub REST calls --- .gitlab/scripts/bump_datadog_agent.py | 81 +++++++++++++++++----- .gitlab/scripts/test_bump_datadog_agent.py | 72 +++++++++++++------ 2 files changed, 115 insertions(+), 38 deletions(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index b174e6f2..bfa35071 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -6,16 +6,24 @@ - sys.argv[1]: the rshell tag (e.g. "v0.0.11") - env GITHUB_TOKEN: a short-lived dd-octo-sts token scoped to DataDog/datadog-agent with contents:write + pull-requests:write. + +Uses the Python standard library only (urllib.request for the GitHub REST API) +so the script runs on any datadog-agent buildimage without pip-install steps. """ from __future__ import annotations +import json import os import re import secrets import subprocess import sys +import urllib.error +import urllib.parse +import urllib.request from pathlib import Path +from typing import Any TARGET_REPO = "DataDog/datadog-agent" TARGET_BASE = "main" @@ -26,15 +34,15 @@ GIT_USER_EMAIL = "github-actions[bot]@users.noreply.github.com" -def run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: - return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) - - def log(msg: str) -> None: """Emit a progress line to stdout. Call sites must never pass secrets.""" print(f"[bump] {msg}", flush=True) +def run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) + + def configure_credentials(workdir: Path, token: str) -> Path: """Store the GitHub token in a local git credentials file under .git/. @@ -81,6 +89,49 @@ def write_release_note(repo_root: Path, version: str) -> Path: return note +class GitHub: + """Minimal GitHub REST client — only the four endpoints this script needs.""" + + def __init__(self, token: str, repo: str): + self._token = token + self._base = f"https://api.github.com/repos/{repo}" + + def _request(self, method: str, path: str, body: dict | None = None) -> Any: + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + f"{self._base}{path}", + data=data, + method=method, + headers={ + "Authorization": f"Bearer {self._token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read() or b"null") + except urllib.error.HTTPError as e: + detail = e.read().decode(errors="replace") + raise RuntimeError(f"GitHub {method} {path} -> {e.code}: {detail}") from None + + def list_open_prs(self, head: str) -> list[dict]: + q = urllib.parse.urlencode({"state": "open", "head": head}) + return self._request("GET", f"/pulls?{q}") + + def create_pull(self, *, title: str, body: str, base: str, head: str, draft: bool = True) -> dict: + return self._request("POST", "/pulls", { + "title": title, "body": body, "base": base, "head": head, "draft": draft, + }) + + def add_labels(self, pr_number: int, labels: list[str]) -> None: + self._request("POST", f"/issues/{pr_number}/labels", {"labels": labels}) + + def request_team_review(self, pr_number: int, team: str) -> None: + self._request("POST", f"/pulls/{pr_number}/requested_reviewers", {"team_reviewers": [team]}) + + def main() -> int: if len(sys.argv) != 2: print("usage: bump_datadog_agent.py ", file=sys.stderr) @@ -96,16 +147,13 @@ def main() -> int: return 1 log(f"preparing bump of {RSHELL_MODULE} to {version}") - from github import Auth, Github, GithubException - - gh = Github(auth=Auth.Token(token), per_page=100) - repo = gh.get_repo(TARGET_REPO) + gh = GitHub(token, TARGET_REPO) branch = f"bump-rshell-{version}" log(f"checking {TARGET_REPO} for existing PR with head={branch}") - existing = list(repo.get_pulls(state="open", head=f"{TARGET_REPO.split('/')[0]}:{branch}")) + existing = gh.list_open_prs(head=f"{TARGET_REPO.split('/')[0]}:{branch}") if existing: - log(f"PR already exists: {existing[0].html_url}; nothing to do") + log(f"PR already exists: {existing[0]['html_url']}; nothing to do") return 0 workdir = Path("/tmp/datadog-agent") @@ -147,7 +195,7 @@ def main() -> int: run(["git", "push", "origin", branch], cwd=workdir) log("opening draft PR") - pr = repo.create_pull( + pr = gh.create_pull( title=f"[automated] Bump rshell to {version}", body=( f"Automated bump of `{RSHELL_MODULE}` to " @@ -157,18 +205,19 @@ def main() -> int: head=branch, draft=True, ) - log(f"opened draft PR: {pr.html_url}") + pr_number = pr["number"] + log(f"opened draft PR: {pr['html_url']}") try: - pr.add_to_labels(*PR_LABELS) + gh.add_labels(pr_number, PR_LABELS) log(f"added labels: {', '.join(PR_LABELS)}") - except GithubException as e: + except Exception as e: log(f"warning: failed to add labels {PR_LABELS}: {e}") try: - pr.create_review_request(team_reviewers=[REVIEW_TEAM]) + gh.request_team_review(pr_number, REVIEW_TEAM) log(f"requested review from @DataDog/{REVIEW_TEAM}") - except GithubException as e: + except Exception as e: log(f"warning: failed to request review from @DataDog/{REVIEW_TEAM}: {e}") return 0 diff --git a/.gitlab/scripts/test_bump_datadog_agent.py b/.gitlab/scripts/test_bump_datadog_agent.py index c8571cba..7612e7c9 100644 --- a/.gitlab/scripts/test_bump_datadog_agent.py +++ b/.gitlab/scripts/test_bump_datadog_agent.py @@ -1,9 +1,9 @@ """Tests for bump_datadog_agent.py. -Runs with stdlib only; PyGithub is stubbed before the script is imported, so -the suite executes anywhere Python 3.10+ is installed: +Runs with stdlib only — the script and its tests have no third-party +dependencies: - python3 -m unittest .gitlab/scripts/test_bump_datadog_agent.py + cd .gitlab/scripts && python3 -m unittest test_bump_datadog_agent -v """ from __future__ import annotations @@ -14,11 +14,7 @@ import textwrap import unittest from pathlib import Path -from unittest.mock import MagicMock, patch - -_github_stub = MagicMock() -_github_stub.GithubException = type("GithubException", (Exception,), {}) -sys.modules["github"] = _github_stub +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).parent)) import bump_datadog_agent as bump # noqa: E402 @@ -191,25 +187,57 @@ def test_missing_github_token_returns_1(self): class TestMainIdempotency(unittest.TestCase): def test_exits_zero_when_pr_already_exists(self): - existing_pr = MagicMock() - existing_pr.html_url = "https://github.com/DataDog/datadog-agent/pull/999" - mock_repo = MagicMock() - mock_repo.get_pulls.return_value = [existing_pr] - _github_stub.Github.reset_mock() - _github_stub.Github.return_value.get_repo.return_value = mock_repo + existing_pr = {"html_url": "https://github.com/DataDog/datadog-agent/pull/999"} with patch.dict(os.environ, {"GITHUB_TOKEN": "fake-token"}): with patch.object(sys, "argv", ["bump_datadog_agent.py", "v0.0.99"]): - # subprocess.run should never be reached on the early-exit path - with patch("bump_datadog_agent.subprocess.run") as mock_run: - result = bump.main() - mock_run.assert_not_called() + with patch.object(bump.GitHub, "list_open_prs", return_value=[existing_pr]) as mock_list: + # subprocess.run should never be reached on the early-exit path + with patch("bump_datadog_agent.subprocess.run") as mock_run: + result = bump.main() + mock_run.assert_not_called() self.assertEqual(result, 0) - mock_repo.get_pulls.assert_called_once() - call_kwargs = mock_repo.get_pulls.call_args.kwargs - self.assertEqual(call_kwargs["state"], "open") - self.assertEqual(call_kwargs["head"], "DataDog:bump-rshell-v0.0.99") + mock_list.assert_called_once_with(head="DataDog:bump-rshell-v0.0.99") + + +class TestGitHubClient(unittest.TestCase): + def test_list_open_prs_builds_correct_url(self): + with patch.object(bump.GitHub, "_request", return_value=[]) as mock_req: + bump.GitHub("tok", "DataDog/datadog-agent").list_open_prs(head="DataDog:my-branch") + mock_req.assert_called_once() + method, path = mock_req.call_args.args[:2] + self.assertEqual(method, "GET") + self.assertIn("state=open", path) + self.assertIn("head=DataDog%3Amy-branch", path) + + def test_create_pull_sends_draft_true_and_body(self): + with patch.object(bump.GitHub, "_request", return_value={"number": 1, "html_url": "x"}) as mock_req: + bump.GitHub("tok", "DataDog/datadog-agent").create_pull( + title="t", body="b", base="main", head="bump-rshell-v0.0.11", draft=True + ) + method, path, body = mock_req.call_args.args + self.assertEqual(method, "POST") + self.assertEqual(path, "/pulls") + self.assertEqual(body["draft"], True) + self.assertEqual(body["title"], "t") + self.assertEqual(body["head"], "bump-rshell-v0.0.11") + + def test_add_labels_hits_issues_endpoint(self): + with patch.object(bump.GitHub, "_request", return_value=None) as mock_req: + bump.GitHub("tok", "DataDog/datadog-agent").add_labels(42, ["foo", "bar"]) + method, path, body = mock_req.call_args.args + self.assertEqual(method, "POST") + self.assertEqual(path, "/issues/42/labels") + self.assertEqual(body, {"labels": ["foo", "bar"]}) + + def test_request_team_review_hits_pulls_endpoint(self): + with patch.object(bump.GitHub, "_request", return_value=None) as mock_req: + bump.GitHub("tok", "DataDog/datadog-agent").request_team_review(42, "action-platform") + method, path, body = mock_req.call_args.args + self.assertEqual(method, "POST") + self.assertEqual(path, "/pulls/42/requested_reviewers") + self.assertEqual(body, {"team_reviewers": ["action-platform"]}) if __name__ == "__main__": From bd57580c52e14abae808479f02d947ca9ee9d3a9 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 15:52:46 -0700 Subject: [PATCH 10/24] Revert "ci(bump): drop PyGithub; use stdlib urllib for GitHub REST calls" This reverts commit 9e0ed9f8cf2fdf8be13a799b04dddd232c20abc2. --- .gitlab/scripts/bump_datadog_agent.py | 81 +++++----------------- .gitlab/scripts/test_bump_datadog_agent.py | 72 ++++++------------- 2 files changed, 38 insertions(+), 115 deletions(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index bfa35071..b174e6f2 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -6,24 +6,16 @@ - sys.argv[1]: the rshell tag (e.g. "v0.0.11") - env GITHUB_TOKEN: a short-lived dd-octo-sts token scoped to DataDog/datadog-agent with contents:write + pull-requests:write. - -Uses the Python standard library only (urllib.request for the GitHub REST API) -so the script runs on any datadog-agent buildimage without pip-install steps. """ from __future__ import annotations -import json import os import re import secrets import subprocess import sys -import urllib.error -import urllib.parse -import urllib.request from pathlib import Path -from typing import Any TARGET_REPO = "DataDog/datadog-agent" TARGET_BASE = "main" @@ -34,15 +26,15 @@ GIT_USER_EMAIL = "github-actions[bot]@users.noreply.github.com" +def run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) + + def log(msg: str) -> None: """Emit a progress line to stdout. Call sites must never pass secrets.""" print(f"[bump] {msg}", flush=True) -def run(cmd: list[str], cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: - return subprocess.run(cmd, cwd=cwd, check=check, text=True, capture_output=True) - - def configure_credentials(workdir: Path, token: str) -> Path: """Store the GitHub token in a local git credentials file under .git/. @@ -89,49 +81,6 @@ def write_release_note(repo_root: Path, version: str) -> Path: return note -class GitHub: - """Minimal GitHub REST client — only the four endpoints this script needs.""" - - def __init__(self, token: str, repo: str): - self._token = token - self._base = f"https://api.github.com/repos/{repo}" - - def _request(self, method: str, path: str, body: dict | None = None) -> Any: - data = json.dumps(body).encode() if body is not None else None - req = urllib.request.Request( - f"{self._base}{path}", - data=data, - method=method, - headers={ - "Authorization": f"Bearer {self._token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - "Content-Type": "application/json", - }, - ) - try: - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read() or b"null") - except urllib.error.HTTPError as e: - detail = e.read().decode(errors="replace") - raise RuntimeError(f"GitHub {method} {path} -> {e.code}: {detail}") from None - - def list_open_prs(self, head: str) -> list[dict]: - q = urllib.parse.urlencode({"state": "open", "head": head}) - return self._request("GET", f"/pulls?{q}") - - def create_pull(self, *, title: str, body: str, base: str, head: str, draft: bool = True) -> dict: - return self._request("POST", "/pulls", { - "title": title, "body": body, "base": base, "head": head, "draft": draft, - }) - - def add_labels(self, pr_number: int, labels: list[str]) -> None: - self._request("POST", f"/issues/{pr_number}/labels", {"labels": labels}) - - def request_team_review(self, pr_number: int, team: str) -> None: - self._request("POST", f"/pulls/{pr_number}/requested_reviewers", {"team_reviewers": [team]}) - - def main() -> int: if len(sys.argv) != 2: print("usage: bump_datadog_agent.py ", file=sys.stderr) @@ -147,13 +96,16 @@ def main() -> int: return 1 log(f"preparing bump of {RSHELL_MODULE} to {version}") - gh = GitHub(token, TARGET_REPO) + from github import Auth, Github, GithubException + + gh = Github(auth=Auth.Token(token), per_page=100) + repo = gh.get_repo(TARGET_REPO) branch = f"bump-rshell-{version}" log(f"checking {TARGET_REPO} for existing PR with head={branch}") - existing = gh.list_open_prs(head=f"{TARGET_REPO.split('/')[0]}:{branch}") + existing = list(repo.get_pulls(state="open", head=f"{TARGET_REPO.split('/')[0]}:{branch}")) if existing: - log(f"PR already exists: {existing[0]['html_url']}; nothing to do") + log(f"PR already exists: {existing[0].html_url}; nothing to do") return 0 workdir = Path("/tmp/datadog-agent") @@ -195,7 +147,7 @@ def main() -> int: run(["git", "push", "origin", branch], cwd=workdir) log("opening draft PR") - pr = gh.create_pull( + pr = repo.create_pull( title=f"[automated] Bump rshell to {version}", body=( f"Automated bump of `{RSHELL_MODULE}` to " @@ -205,19 +157,18 @@ def main() -> int: head=branch, draft=True, ) - pr_number = pr["number"] - log(f"opened draft PR: {pr['html_url']}") + log(f"opened draft PR: {pr.html_url}") try: - gh.add_labels(pr_number, PR_LABELS) + pr.add_to_labels(*PR_LABELS) log(f"added labels: {', '.join(PR_LABELS)}") - except Exception as e: + except GithubException as e: log(f"warning: failed to add labels {PR_LABELS}: {e}") try: - gh.request_team_review(pr_number, REVIEW_TEAM) + pr.create_review_request(team_reviewers=[REVIEW_TEAM]) log(f"requested review from @DataDog/{REVIEW_TEAM}") - except Exception as e: + except GithubException as e: log(f"warning: failed to request review from @DataDog/{REVIEW_TEAM}: {e}") return 0 diff --git a/.gitlab/scripts/test_bump_datadog_agent.py b/.gitlab/scripts/test_bump_datadog_agent.py index 7612e7c9..c8571cba 100644 --- a/.gitlab/scripts/test_bump_datadog_agent.py +++ b/.gitlab/scripts/test_bump_datadog_agent.py @@ -1,9 +1,9 @@ """Tests for bump_datadog_agent.py. -Runs with stdlib only — the script and its tests have no third-party -dependencies: +Runs with stdlib only; PyGithub is stubbed before the script is imported, so +the suite executes anywhere Python 3.10+ is installed: - cd .gitlab/scripts && python3 -m unittest test_bump_datadog_agent -v + python3 -m unittest .gitlab/scripts/test_bump_datadog_agent.py """ from __future__ import annotations @@ -14,7 +14,11 @@ import textwrap import unittest from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch + +_github_stub = MagicMock() +_github_stub.GithubException = type("GithubException", (Exception,), {}) +sys.modules["github"] = _github_stub sys.path.insert(0, str(Path(__file__).parent)) import bump_datadog_agent as bump # noqa: E402 @@ -187,57 +191,25 @@ def test_missing_github_token_returns_1(self): class TestMainIdempotency(unittest.TestCase): def test_exits_zero_when_pr_already_exists(self): - existing_pr = {"html_url": "https://github.com/DataDog/datadog-agent/pull/999"} + existing_pr = MagicMock() + existing_pr.html_url = "https://github.com/DataDog/datadog-agent/pull/999" + mock_repo = MagicMock() + mock_repo.get_pulls.return_value = [existing_pr] + _github_stub.Github.reset_mock() + _github_stub.Github.return_value.get_repo.return_value = mock_repo with patch.dict(os.environ, {"GITHUB_TOKEN": "fake-token"}): with patch.object(sys, "argv", ["bump_datadog_agent.py", "v0.0.99"]): - with patch.object(bump.GitHub, "list_open_prs", return_value=[existing_pr]) as mock_list: - # subprocess.run should never be reached on the early-exit path - with patch("bump_datadog_agent.subprocess.run") as mock_run: - result = bump.main() - mock_run.assert_not_called() + # subprocess.run should never be reached on the early-exit path + with patch("bump_datadog_agent.subprocess.run") as mock_run: + result = bump.main() + mock_run.assert_not_called() self.assertEqual(result, 0) - mock_list.assert_called_once_with(head="DataDog:bump-rshell-v0.0.99") - - -class TestGitHubClient(unittest.TestCase): - def test_list_open_prs_builds_correct_url(self): - with patch.object(bump.GitHub, "_request", return_value=[]) as mock_req: - bump.GitHub("tok", "DataDog/datadog-agent").list_open_prs(head="DataDog:my-branch") - mock_req.assert_called_once() - method, path = mock_req.call_args.args[:2] - self.assertEqual(method, "GET") - self.assertIn("state=open", path) - self.assertIn("head=DataDog%3Amy-branch", path) - - def test_create_pull_sends_draft_true_and_body(self): - with patch.object(bump.GitHub, "_request", return_value={"number": 1, "html_url": "x"}) as mock_req: - bump.GitHub("tok", "DataDog/datadog-agent").create_pull( - title="t", body="b", base="main", head="bump-rshell-v0.0.11", draft=True - ) - method, path, body = mock_req.call_args.args - self.assertEqual(method, "POST") - self.assertEqual(path, "/pulls") - self.assertEqual(body["draft"], True) - self.assertEqual(body["title"], "t") - self.assertEqual(body["head"], "bump-rshell-v0.0.11") - - def test_add_labels_hits_issues_endpoint(self): - with patch.object(bump.GitHub, "_request", return_value=None) as mock_req: - bump.GitHub("tok", "DataDog/datadog-agent").add_labels(42, ["foo", "bar"]) - method, path, body = mock_req.call_args.args - self.assertEqual(method, "POST") - self.assertEqual(path, "/issues/42/labels") - self.assertEqual(body, {"labels": ["foo", "bar"]}) - - def test_request_team_review_hits_pulls_endpoint(self): - with patch.object(bump.GitHub, "_request", return_value=None) as mock_req: - bump.GitHub("tok", "DataDog/datadog-agent").request_team_review(42, "action-platform") - method, path, body = mock_req.call_args.args - self.assertEqual(method, "POST") - self.assertEqual(path, "/pulls/42/requested_reviewers") - self.assertEqual(body, {"team_reviewers": ["action-platform"]}) + mock_repo.get_pulls.assert_called_once() + call_kwargs = mock_repo.get_pulls.call_args.kwargs + self.assertEqual(call_kwargs["state"], "open") + self.assertEqual(call_kwargs["head"], "DataDog:bump-rshell-v0.0.99") if __name__ == "__main__": From 20d3ba3fda06e35f2719de69d58819e342610126 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 15:53:06 -0700 Subject: [PATCH 11/24] ci(bump): install PyGithub in before_script --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32138b2d..a1608085 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,5 +21,6 @@ bump_datadog_agent: - if: $CI_PIPELINE_SOURCE == "web" && $BUMP_VERSION =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ before_script: - !reference [.setup_github_bump_token] + - pip install PyGithub script: - python3 .gitlab/scripts/bump_datadog_agent.py "${CI_COMMIT_TAG:-$BUMP_VERSION}" From 9cf0d688a76959179dd054a9a3e70626124ae111 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:00:01 -0700 Subject: [PATCH 12/24] ci(bump): restrict web-triggered bump to protected main branch --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1608085..0b3e1a35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ bump_datadog_agent: resource_group: rshell-bump rules: - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ - - if: $CI_PIPELINE_SOURCE == "web" && $BUMP_VERSION =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ + - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == "main" && $CI_COMMIT_REF_PROTECTED == "true" && $BUMP_VERSION =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ before_script: - !reference [.setup_github_bump_token] - pip install PyGithub From 1b0b333182202f50fbcfbfb67e54cd72f9779f9e Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:02:46 -0700 Subject: [PATCH 13/24] ci(bump): drop redundant ref_protected check (main is always protected) --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b3e1a35..7438de48 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ bump_datadog_agent: resource_group: rshell-bump rules: - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == "main" && $CI_COMMIT_REF_PROTECTED == "true" && $BUMP_VERSION =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ + - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == "main" && $BUMP_VERSION =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ before_script: - !reference [.setup_github_bump_token] - pip install PyGithub From 063390569880f61fa2176a00d3a0756a3af4a7e9 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:04:16 -0700 Subject: [PATCH 14/24] ci(bump): skip bump flow when target version is already pinned --- .gitlab/scripts/bump_datadog_agent.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index b174e6f2..683084ee 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -122,20 +122,27 @@ def main() -> int: go_mod = workdir / "go.mod" previous_version = current_rshell_version(go_mod) log(f"current pinned version in go.mod: {previous_version or ''}") + + if previous_version == version: + log(f"datadog-agent already pins rshell at {version}; nothing to do") + return 0 + strip_rshell_replace(go_mod) log(f"running: go get {RSHELL_MODULE}@{version}") run(["go", "get", f"{RSHELL_MODULE}@{version}"], cwd=workdir) log("running: dda inv tidy") run(["dda", "inv", "tidy"], cwd=workdir) - note = write_release_note(workdir, version) - log(f"wrote release note: {note.relative_to(workdir)}") run(["git", "add", "-A"], cwd=workdir) diff = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=workdir) if diff.returncode == 0: - log(f"no staged changes; datadog-agent already at rshell {version}") + log(f"no changes to go.mod/go.sum; datadog-agent already at rshell {version}") return 0 + note = write_release_note(workdir, version) + log(f"wrote release note: {note.relative_to(workdir)}") + run(["git", "add", str(note)], cwd=workdir) + commit_msg = ( f"Bump rshell dependency from {previous_version} to {version}" if previous_version From 3c0ac3884b597047f870f3c1eb056e758450c95d Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:07:11 -0700 Subject: [PATCH 15/24] ci(bump): make release-note filename deterministic per version --- .gitlab/scripts/bump_datadog_agent.py | 5 +++-- .gitlab/scripts/test_bump_datadog_agent.py | 10 +++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index 683084ee..a1738e57 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -10,9 +10,9 @@ from __future__ import annotations +import hashlib import os import re -import secrets import subprocess import sys from pathlib import Path @@ -70,7 +70,8 @@ def current_rshell_version(go_mod: Path) -> str | None: def write_release_note(repo_root: Path, version: str) -> Path: - suffix = secrets.token_hex(8) + # Deterministic per (module, version) so retries produce the identical file. + suffix = hashlib.sha256(f"{RSHELL_MODULE}@{version}".encode()).hexdigest()[:16] note = repo_root / "releasenotes" / "notes" / f"bump-rshell-{version}-{suffix}.yaml" note.write_text( "---\n" diff --git a/.gitlab/scripts/test_bump_datadog_agent.py b/.gitlab/scripts/test_bump_datadog_agent.py index c8571cba..04d0dd8f 100644 --- a/.gitlab/scripts/test_bump_datadog_agent.py +++ b/.gitlab/scripts/test_bump_datadog_agent.py @@ -155,12 +155,20 @@ def test_writes_correct_format_and_path(self): self.assertIn("enhancements:", content) self.assertIn("Bump ``rshell`` to v0.0.11", content) - def test_filenames_are_unique(self): + def test_filename_is_deterministic_for_same_version(self): with tempfile.TemporaryDirectory() as td: repo_root = Path(td) (repo_root / "releasenotes" / "notes").mkdir(parents=True) a = bump.write_release_note(repo_root, "v0.0.11") b = bump.write_release_note(repo_root, "v0.0.11") + self.assertEqual(a.name, b.name) + + def test_filename_differs_between_versions(self): + with tempfile.TemporaryDirectory() as td: + repo_root = Path(td) + (repo_root / "releasenotes" / "notes").mkdir(parents=True) + a = bump.write_release_note(repo_root, "v0.0.11") + b = bump.write_release_note(repo_root, "v0.0.12") self.assertNotEqual(a.name, b.name) From 5ecc8eba8f72a974db2a240400738cbe290865da Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:10:02 -0700 Subject: [PATCH 16/24] ci(bump): force-push bot branch to enable retries after partial failure --- .gitlab/scripts/bump_datadog_agent.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index a1738e57..f889b0b0 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -151,8 +151,11 @@ def main() -> int: ) log(f"committing: {commit_msg}") run(["git", "commit", "-m", commit_msg], cwd=workdir) - log(f"pushing branch {branch} to origin") - run(["git", "push", "origin", branch], cwd=workdir) + log(f"pushing branch {branch} to origin (force)") + # Force push is safe: this branch is only ever written by this script, and + # the force handles retries after a prior failure (deterministic tree, + # non-deterministic commit timestamp). + run(["git", "push", "--force", "origin", branch], cwd=workdir) log("opening draft PR") pr = repo.create_pull( From 5f4aebdbf60deb6a86c0eae0367e38135dc922e3 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:12:58 -0700 Subject: [PATCH 17/24] ci(bump): install deps before minting GitHub token --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7438de48..84abb62f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,9 @@ bump_datadog_agent: - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == "main" && $BUMP_VERSION =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ before_script: - - !reference [.setup_github_bump_token] + # Install dependencies BEFORE minting the GITHUB_TOKEN so a compromised + # package's install-time hook can't read the token from the environment. - pip install PyGithub + - !reference [.setup_github_bump_token] script: - python3 .gitlab/scripts/bump_datadog_agent.py "${CI_COMMIT_TAG:-$BUMP_VERSION}" From b363e858971a2e8857e394d87f14d6e137669941 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:15:42 -0700 Subject: [PATCH 18/24] ci(bump): strip rshell replace directives in all valid go.mod forms --- .gitlab/scripts/bump_datadog_agent.py | 22 +++++- .gitlab/scripts/test_bump_datadog_agent.py | 91 ++++++++++++++++++++++ 2 files changed, 109 insertions(+), 4 deletions(-) diff --git a/.gitlab/scripts/bump_datadog_agent.py b/.gitlab/scripts/bump_datadog_agent.py index f889b0b0..fafd2fe5 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/.gitlab/scripts/bump_datadog_agent.py @@ -49,14 +49,28 @@ def configure_credentials(workdir: Path, token: str) -> Path: return creds_path +_RSHELL_REPLACE_RE = re.compile( + rf"^[ \t]*(?:replace\s+)?{re.escape(RSHELL_MODULE)}(?:\s+v\S+)?\s+=>\s+[^\n]*$\n?", + re.MULTILINE, +) + + def strip_rshell_replace(go_mod: Path) -> None: - """Drop any `replace github.com/DataDog/rshell => ...` line (one-time v0.0.11 transition).""" + """Remove rshell replace directives from go.mod in every valid Go form. + + Handles: + - single-line unversioned: replace github.com/DataDog/rshell => /path + - single-line versioned: replace github.com/DataDog/rshell v0.0.10 => /path + - block-form entries (no leading `replace` keyword on the line itself) + + Any leftover empty `replace ( )` block is normalized away by `dda inv tidy` + downstream. + """ original = go_mod.read_text() - pattern = re.compile(rf"^\s*replace\s+{re.escape(RSHELL_MODULE)}\s+=>.*$\n?", re.MULTILINE) - updated = pattern.sub("", original) + updated = _RSHELL_REPLACE_RE.sub("", original) if updated != original: go_mod.write_text(updated) - log(f"stripped `replace {RSHELL_MODULE} =>` directive from go.mod") + log(f"stripped replace directive(s) for {RSHELL_MODULE} from go.mod") def current_rshell_version(go_mod: Path) -> str | None: diff --git a/.gitlab/scripts/test_bump_datadog_agent.py b/.gitlab/scripts/test_bump_datadog_agent.py index 04d0dd8f..49492d14 100644 --- a/.gitlab/scripts/test_bump_datadog_agent.py +++ b/.gitlab/scripts/test_bump_datadog_agent.py @@ -99,6 +99,97 @@ def test_preserves_other_replaces(self): self.assertIn("github.com/other/mod", updated) self.assertIn("github.com/yet/another", updated) + def test_removes_single_line_versioned_replace(self): + original = textwrap.dedent( + """\ + module m + go 1.25 + + replace github.com/DataDog/rshell v0.0.10 => /local/path + + require github.com/DataDog/rshell v0.0.10 + """ + ) + with tempfile.TemporaryDirectory() as td: + go_mod = Path(td) / "go.mod" + go_mod.write_text(original) + bump.strip_rshell_replace(go_mod) + updated = go_mod.read_text() + self.assertNotIn("replace github.com/DataDog/rshell", updated) + self.assertIn("require github.com/DataDog/rshell v0.0.10", updated) + + def test_removes_block_form_unversioned_entry(self): + original = textwrap.dedent( + """\ + module m + go 1.25 + + replace ( + github.com/DataDog/rshell => /local/path + ) + """ + ) + with tempfile.TemporaryDirectory() as td: + go_mod = Path(td) / "go.mod" + go_mod.write_text(original) + bump.strip_rshell_replace(go_mod) + updated = go_mod.read_text() + self.assertNotIn("DataDog/rshell", updated) + + def test_removes_block_form_versioned_entry(self): + original = textwrap.dedent( + """\ + module m + go 1.25 + + replace ( + github.com/DataDog/rshell v0.0.10 => /local/path + ) + """ + ) + with tempfile.TemporaryDirectory() as td: + go_mod = Path(td) / "go.mod" + go_mod.write_text(original) + bump.strip_rshell_replace(go_mod) + updated = go_mod.read_text() + self.assertNotIn("DataDog/rshell", updated) + + def test_strips_only_rshell_entries_from_mixed_block(self): + original = textwrap.dedent( + """\ + replace ( + github.com/other => /a + github.com/DataDog/rshell => /rshell-local + github.com/DataDog/rshell v0.0.10 => /rshell-pinned + github.com/another => /b + ) + """ + ) + with tempfile.TemporaryDirectory() as td: + go_mod = Path(td) / "go.mod" + go_mod.write_text(original) + bump.strip_rshell_replace(go_mod) + updated = go_mod.read_text() + self.assertNotIn("DataDog/rshell", updated) + self.assertIn("github.com/other => /a", updated) + self.assertIn("github.com/another => /b", updated) + + def test_does_not_touch_require_lines(self): + # Lines in a require block look similar but have no `=>`; must not be + # stripped. + original = textwrap.dedent( + """\ + require ( + github.com/DataDog/rshell v0.0.10 + ) + """ + ) + with tempfile.TemporaryDirectory() as td: + go_mod = Path(td) / "go.mod" + go_mod.write_text(original) + bump.strip_rshell_replace(go_mod) + self.assertEqual(go_mod.read_text(), original) + class TestCurrentRshellVersion(unittest.TestCase): def _write(self, content: str) -> Path: From b15a36f9de9e6d2f1a3e8cd410ee38c5317b6cbe Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:23:27 -0700 Subject: [PATCH 19/24] refactor: move bump script to tools/ and add GitHub Actions test workflow --- .github/workflows/test-ci-scripts.yml | 26 +++++++++++++++++++ .gitlab-ci.yml | 2 +- .../bump_datadog_agent/bump.py | 2 +- .../bump_datadog_agent/tests/test_bump.py | 16 +++++++----- 4 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/test-ci-scripts.yml rename .gitlab/scripts/bump_datadog_agent.py => tools/bump_datadog_agent/bump.py (99%) rename .gitlab/scripts/test_bump_datadog_agent.py => tools/bump_datadog_agent/tests/test_bump.py (95%) diff --git a/.github/workflows/test-ci-scripts.yml b/.github/workflows/test-ci-scripts.yml new file mode 100644 index 00000000..95df8c2a --- /dev/null +++ b/.github/workflows/test-ci-scripts.yml @@ -0,0 +1,26 @@ +name: CI scripts + +on: + push: + branches: [main] + paths: + - 'tools/**' + - '.github/workflows/test-ci-scripts.yml' + pull_request: + branches: [main] + paths: + - 'tools/**' + - '.github/workflows/test-ci-scripts.yml' + +permissions: + contents: read + +jobs: + test-bump-datadog-agent: + name: Test bump_datadog_agent + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Run unit tests + run: python3 -m unittest discover -s tools/bump_datadog_agent/tests -v diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 84abb62f..4ad6d3de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,4 +25,4 @@ bump_datadog_agent: - pip install PyGithub - !reference [.setup_github_bump_token] script: - - python3 .gitlab/scripts/bump_datadog_agent.py "${CI_COMMIT_TAG:-$BUMP_VERSION}" + - python3 tools/bump_datadog_agent/bump.py "${CI_COMMIT_TAG:-$BUMP_VERSION}" diff --git a/.gitlab/scripts/bump_datadog_agent.py b/tools/bump_datadog_agent/bump.py similarity index 99% rename from .gitlab/scripts/bump_datadog_agent.py rename to tools/bump_datadog_agent/bump.py index fafd2fe5..e4f3b327 100755 --- a/.gitlab/scripts/bump_datadog_agent.py +++ b/tools/bump_datadog_agent/bump.py @@ -98,7 +98,7 @@ def write_release_note(repo_root: Path, version: str) -> Path: def main() -> int: if len(sys.argv) != 2: - print("usage: bump_datadog_agent.py ", file=sys.stderr) + print("usage: bump.py ", file=sys.stderr) return 2 version = sys.argv[1] if not re.fullmatch(r"v\d+\.\d+\.\d+", version): diff --git a/.gitlab/scripts/test_bump_datadog_agent.py b/tools/bump_datadog_agent/tests/test_bump.py similarity index 95% rename from .gitlab/scripts/test_bump_datadog_agent.py rename to tools/bump_datadog_agent/tests/test_bump.py index 49492d14..d8c9047d 100644 --- a/.gitlab/scripts/test_bump_datadog_agent.py +++ b/tools/bump_datadog_agent/tests/test_bump.py @@ -1,9 +1,9 @@ -"""Tests for bump_datadog_agent.py. +"""Tests for tools/bump_datadog_agent/bump.py. Runs with stdlib only; PyGithub is stubbed before the script is imported, so -the suite executes anywhere Python 3.10+ is installed: +the suite executes anywhere Python 3.10+ is installed. From the repo root: - python3 -m unittest .gitlab/scripts/test_bump_datadog_agent.py + python3 -m unittest discover -s tools/bump_datadog_agent/tests -v """ from __future__ import annotations @@ -20,8 +20,10 @@ _github_stub.GithubException = type("GithubException", (Exception,), {}) sys.modules["github"] = _github_stub -sys.path.insert(0, str(Path(__file__).parent)) -import bump_datadog_agent as bump # noqa: E402 +# Put the script's directory on sys.path so we can `import bump` regardless of +# where the test runner is invoked from. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import bump # noqa: E402 class TestConfigureCredentials(unittest.TestCase): @@ -29,7 +31,7 @@ def test_writes_credentials_file_and_invokes_git_config(self): with tempfile.TemporaryDirectory() as td: workdir = Path(td) (workdir / ".git").mkdir() - with patch("bump_datadog_agent.subprocess.run") as mock_run: + with patch("bump.subprocess.run") as mock_run: mock_run.return_value.returncode = 0 path = bump.configure_credentials(workdir, "ghs_SECRET123") @@ -300,7 +302,7 @@ def test_exits_zero_when_pr_already_exists(self): with patch.dict(os.environ, {"GITHUB_TOKEN": "fake-token"}): with patch.object(sys, "argv", ["bump_datadog_agent.py", "v0.0.99"]): # subprocess.run should never be reached on the early-exit path - with patch("bump_datadog_agent.subprocess.run") as mock_run: + with patch("bump.subprocess.run") as mock_run: result = bump.main() mock_run.assert_not_called() From 9ae59eed60a551e9b501fba8b37f4f57b31f858e Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:26:47 -0700 Subject: [PATCH 20/24] ci(bump): pin PyGithub to a specific version for reproducibility --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ad6d3de..b4daf9b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,7 +22,8 @@ bump_datadog_agent: before_script: # Install dependencies BEFORE minting the GITHUB_TOKEN so a compromised # package's install-time hook can't read the token from the environment. - - pip install PyGithub + # Pin to a vetted PyGithub version; review periodically and bump deliberately. + - pip install "PyGithub==2.5.0" - !reference [.setup_github_bump_token] script: - python3 tools/bump_datadog_agent/bump.py "${CI_COMMIT_TAG:-$BUMP_VERSION}" From 47b1847a6e59802b27567dfda6f7e56421b9cda1 Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:29:59 -0700 Subject: [PATCH 21/24] ci(bump): use unique tempdir per run instead of hard-coded /tmp path --- tools/bump_datadog_agent/bump.py | 99 +++++++++++++++++--------------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/tools/bump_datadog_agent/bump.py b/tools/bump_datadog_agent/bump.py index e4f3b327..9033c5fa 100755 --- a/tools/bump_datadog_agent/bump.py +++ b/tools/bump_datadog_agent/bump.py @@ -15,6 +15,7 @@ import re import subprocess import sys +import tempfile from pathlib import Path TARGET_REPO = "DataDog/datadog-agent" @@ -123,53 +124,57 @@ def main() -> int: log(f"PR already exists: {existing[0].html_url}; nothing to do") return 0 - workdir = Path("/tmp/datadog-agent") - clone_url = f"https://github.com/{TARGET_REPO}.git" - log(f"cloning {TARGET_REPO}@{TARGET_BASE} into {workdir}") - run(["git", "clone", "--depth=1", "--branch", TARGET_BASE, clone_url, str(workdir)]) - log("configuring git credentials (token stored in .git/ci-credentials, not argv)") - configure_credentials(workdir, token) - run(["git", "config", "user.name", GIT_USER_NAME], cwd=workdir) - run(["git", "config", "user.email", GIT_USER_EMAIL], cwd=workdir) - log(f"creating branch {branch}") - run(["git", "checkout", "-b", branch], cwd=workdir) - - go_mod = workdir / "go.mod" - previous_version = current_rshell_version(go_mod) - log(f"current pinned version in go.mod: {previous_version or ''}") - - if previous_version == version: - log(f"datadog-agent already pins rshell at {version}; nothing to do") - return 0 - - strip_rshell_replace(go_mod) - log(f"running: go get {RSHELL_MODULE}@{version}") - run(["go", "get", f"{RSHELL_MODULE}@{version}"], cwd=workdir) - log("running: dda inv tidy") - run(["dda", "inv", "tidy"], cwd=workdir) - - run(["git", "add", "-A"], cwd=workdir) - diff = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=workdir) - if diff.returncode == 0: - log(f"no changes to go.mod/go.sum; datadog-agent already at rshell {version}") - return 0 - - note = write_release_note(workdir, version) - log(f"wrote release note: {note.relative_to(workdir)}") - run(["git", "add", str(note)], cwd=workdir) - - commit_msg = ( - f"Bump rshell dependency from {previous_version} to {version}" - if previous_version - else f"Bump rshell dependency to {version}" - ) - log(f"committing: {commit_msg}") - run(["git", "commit", "-m", commit_msg], cwd=workdir) - log(f"pushing branch {branch} to origin (force)") - # Force push is safe: this branch is only ever written by this script, and - # the force handles retries after a prior failure (deterministic tree, - # non-deterministic commit timestamp). - run(["git", "push", "--force", "origin", branch], cwd=workdir) + # Use a fresh, unique tempdir each run. Auto-cleanup on exit means no stale + # state leaks between runs; starting empty means `git clone` never fails + # with exit 128 because a prior directory already existed. + with tempfile.TemporaryDirectory(prefix="bump-datadog-agent-") as td: + workdir = Path(td) / "datadog-agent" + clone_url = f"https://github.com/{TARGET_REPO}.git" + log(f"cloning {TARGET_REPO}@{TARGET_BASE} into {workdir}") + run(["git", "clone", "--depth=1", "--branch", TARGET_BASE, clone_url, str(workdir)]) + log("configuring git credentials (token stored in .git/ci-credentials, not argv)") + configure_credentials(workdir, token) + run(["git", "config", "user.name", GIT_USER_NAME], cwd=workdir) + run(["git", "config", "user.email", GIT_USER_EMAIL], cwd=workdir) + log(f"creating branch {branch}") + run(["git", "checkout", "-b", branch], cwd=workdir) + + go_mod = workdir / "go.mod" + previous_version = current_rshell_version(go_mod) + log(f"current pinned version in go.mod: {previous_version or ''}") + + if previous_version == version: + log(f"datadog-agent already pins rshell at {version}; nothing to do") + return 0 + + strip_rshell_replace(go_mod) + log(f"running: go get {RSHELL_MODULE}@{version}") + run(["go", "get", f"{RSHELL_MODULE}@{version}"], cwd=workdir) + log("running: dda inv tidy") + run(["dda", "inv", "tidy"], cwd=workdir) + + run(["git", "add", "-A"], cwd=workdir) + diff = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=workdir) + if diff.returncode == 0: + log(f"no changes to go.mod/go.sum; datadog-agent already at rshell {version}") + return 0 + + note = write_release_note(workdir, version) + log(f"wrote release note: {note.relative_to(workdir)}") + run(["git", "add", str(note)], cwd=workdir) + + commit_msg = ( + f"Bump rshell dependency from {previous_version} to {version}" + if previous_version + else f"Bump rshell dependency to {version}" + ) + log(f"committing: {commit_msg}") + run(["git", "commit", "-m", commit_msg], cwd=workdir) + log(f"pushing branch {branch} to origin (force)") + # Force push is safe: this branch is only ever written by this script, and + # the force handles retries after a prior failure (deterministic tree, + # non-deterministic commit timestamp). + run(["git", "push", "--force", "origin", branch], cwd=workdir) log("opening draft PR") pr = repo.create_pull( From 810ac250e083145b3fe0343b1c6f1e30007475ff Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Thu, 16 Apr 2026 16:36:35 -0700 Subject: [PATCH 22/24] ci(bump): inline token export instead of !reference template --- .gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4daf9b3..0415a1e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,9 +7,6 @@ stages: DDOCTOSTS_ID_TOKEN: aud: dd-octo-sts -.setup_github_bump_token: - - export GITHUB_TOKEN=$(dd-octo-sts token --scope DataDog/datadog-agent --policy self.rshell.bump-rshell-version) - bump_datadog_agent: stage: trigger_release image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux:latest @@ -24,6 +21,6 @@ bump_datadog_agent: # package's install-time hook can't read the token from the environment. # Pin to a vetted PyGithub version; review periodically and bump deliberately. - pip install "PyGithub==2.5.0" - - !reference [.setup_github_bump_token] + - export GITHUB_TOKEN=$(dd-octo-sts token --scope DataDog/datadog-agent --policy self.rshell.bump-rshell-version) script: - python3 tools/bump_datadog_agent/bump.py "${CI_COMMIT_TAG:-$BUMP_VERSION}" From ef39f1508219bbbcca88d2c10be442a9325614aa Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Fri, 17 Apr 2026 08:00:20 -0700 Subject: [PATCH 23/24] ci(bump): scrub GITHUB_TOKEN from env before spawning subprocesses --- tools/bump_datadog_agent/bump.py | 9 +++++++++ tools/bump_datadog_agent/tests/test_bump.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tools/bump_datadog_agent/bump.py b/tools/bump_datadog_agent/bump.py index 9033c5fa..3caaf503 100755 --- a/tools/bump_datadog_agent/bump.py +++ b/tools/bump_datadog_agent/bump.py @@ -118,6 +118,15 @@ def main() -> int: repo = gh.get_repo(TARGET_REPO) branch = f"bump-rshell-{version}" + # Scrub the token from the process environment now that PyGithub has + # internalized it. Subsequent subprocess calls (git, go, dda inv tidy, + # which executes code from the freshly cloned repo) inherit os.environ by + # default; leaving the write-scoped GITHUB_TOKEN in that environment would + # let any of them exfiltrate it. PyGithub still holds the token in its + # auth object, and git push authenticates via the on-disk credential-helper + # file rather than the env var. + os.environ.pop("GITHUB_TOKEN", None) + log(f"checking {TARGET_REPO} for existing PR with head={branch}") existing = list(repo.get_pulls(state="open", head=f"{TARGET_REPO.split('/')[0]}:{branch}")) if existing: diff --git a/tools/bump_datadog_agent/tests/test_bump.py b/tools/bump_datadog_agent/tests/test_bump.py index d8c9047d..e4aaabf7 100644 --- a/tools/bump_datadog_agent/tests/test_bump.py +++ b/tools/bump_datadog_agent/tests/test_bump.py @@ -313,5 +313,24 @@ def test_exits_zero_when_pr_already_exists(self): self.assertEqual(call_kwargs["head"], "DataDog:bump-rshell-v0.0.99") +class TestTokenScrubbing(unittest.TestCase): + def test_github_token_removed_from_environ_before_subprocess_calls(self): + # Use the "PR already exists" path because it goes just far enough to + # create the GitHub client — which is where the token should get + # scrubbed — without needing to mock clone/go/dda. + existing_pr = MagicMock() + existing_pr.html_url = "https://github.com/DataDog/datadog-agent/pull/1" + mock_repo = MagicMock() + mock_repo.get_pulls.return_value = [existing_pr] + _github_stub.Github.reset_mock() + _github_stub.Github.return_value.get_repo.return_value = mock_repo + + with patch.dict(os.environ, {"GITHUB_TOKEN": "fake-token"}, clear=False): + with patch.object(sys, "argv", ["bump.py", "v0.0.99"]): + bump.main() + # After main(), the token must no longer be readable from env. + self.assertNotIn("GITHUB_TOKEN", os.environ) + + if __name__ == "__main__": unittest.main() From 8537f5e5c582aeb37deb9a5cfa4a662a4e42716f Mon Sep 17 00:00:00 2001 From: Matthew DeGuzman Date: Fri, 17 Apr 2026 08:23:49 -0700 Subject: [PATCH 24/24] ci(bump): handle closed/merged PRs; reopen instead of duplicate-creating --- tools/bump_datadog_agent/bump.py | 54 ++++++++++++++------ tools/bump_datadog_agent/tests/test_bump.py | 56 ++++++++++++++++----- 2 files changed, 82 insertions(+), 28 deletions(-) diff --git a/tools/bump_datadog_agent/bump.py b/tools/bump_datadog_agent/bump.py index 3caaf503..e6a969ed 100755 --- a/tools/bump_datadog_agent/bump.py +++ b/tools/bump_datadog_agent/bump.py @@ -127,12 +127,27 @@ def main() -> int: # file rather than the env var. os.environ.pop("GITHUB_TOKEN", None) - log(f"checking {TARGET_REPO} for existing PR with head={branch}") - existing = list(repo.get_pulls(state="open", head=f"{TARGET_REPO.split('/')[0]}:{branch}")) - if existing: - log(f"PR already exists: {existing[0].html_url}; nothing to do") + # Include closed + merged PRs too, so we can distinguish: + # - open: stop (human owns the cycle) + # - merged: log and let the go.mod/go.sum diff decide + # - closed unmerged: reopen it at create-pull time instead of failing + log(f"checking {TARGET_REPO} for existing PRs with head={branch}") + all_existing = list( + repo.get_pulls(state="all", head=f"{TARGET_REPO.split('/')[0]}:{branch}") + ) + open_existing = [p for p in all_existing if p.state == "open"] + if open_existing: + log(f"open PR already exists: {open_existing[0].html_url}; nothing to do") return 0 + closed_unmerged = [p for p in all_existing if p.state == "closed" and not p.merged] + merged_existing = [p for p in all_existing if p.merged] + if merged_existing: + log( + f"prior PR merged ({merged_existing[0].html_url}); will no-op unless " + "go.mod/go.sum shows a genuine diff" + ) + # Use a fresh, unique tempdir each run. Auto-cleanup on exit means no stale # state leaks between runs; starting empty means `git clone` never fails # with exit 128 because a prior directory already existed. @@ -185,18 +200,27 @@ def main() -> int: # non-deterministic commit timestamp). run(["git", "push", "--force", "origin", branch], cwd=workdir) - log("opening draft PR") - pr = repo.create_pull( - title=f"[automated] Bump rshell to {version}", - body=( - f"Automated bump of `{RSHELL_MODULE}` to " - f"[{version}](https://github.com/DataDog/rshell/releases/tag/{version}).\n" - ), - base=TARGET_BASE, - head=branch, - draft=True, + pr_title = f"[automated] Bump rshell to {version}" + pr_body = ( + f"Automated bump of `{RSHELL_MODULE}` to " + f"[{version}](https://github.com/DataDog/rshell/releases/tag/{version}).\n" ) - log(f"opened draft PR: {pr.html_url}") + if closed_unmerged: + # Reusing a previously-closed PR avoids GitHub's duplicate-PR error and + # keeps the review history in one thread. + pr = closed_unmerged[0] + log(f"reopening prior closed PR: {pr.html_url}") + pr.edit(state="open", title=pr_title, body=pr_body) + else: + log("opening draft PR") + pr = repo.create_pull( + title=pr_title, + body=pr_body, + base=TARGET_BASE, + head=branch, + draft=True, + ) + log(f"opened draft PR: {pr.html_url}") try: pr.add_to_labels(*PR_LABELS) diff --git a/tools/bump_datadog_agent/tests/test_bump.py b/tools/bump_datadog_agent/tests/test_bump.py index e4aaabf7..ab196b01 100644 --- a/tools/bump_datadog_agent/tests/test_bump.py +++ b/tools/bump_datadog_agent/tests/test_bump.py @@ -291,35 +291,65 @@ def test_missing_github_token_returns_1(self): class TestMainIdempotency(unittest.TestCase): - def test_exits_zero_when_pr_already_exists(self): - existing_pr = MagicMock() - existing_pr.html_url = "https://github.com/DataDog/datadog-agent/pull/999" + def _setup_pulls(self, prs): mock_repo = MagicMock() - mock_repo.get_pulls.return_value = [existing_pr] + mock_repo.get_pulls.return_value = prs _github_stub.Github.reset_mock() _github_stub.Github.return_value.get_repo.return_value = mock_repo + return mock_repo + + def test_exits_zero_when_open_pr_already_exists(self): + existing_pr = MagicMock() + existing_pr.state = "open" + existing_pr.merged = False + existing_pr.html_url = "https://github.com/DataDog/datadog-agent/pull/999" + mock_repo = self._setup_pulls([existing_pr]) with patch.dict(os.environ, {"GITHUB_TOKEN": "fake-token"}): - with patch.object(sys, "argv", ["bump_datadog_agent.py", "v0.0.99"]): - # subprocess.run should never be reached on the early-exit path + with patch.object(sys, "argv", ["bump.py", "v0.0.99"]): with patch("bump.subprocess.run") as mock_run: result = bump.main() mock_run.assert_not_called() self.assertEqual(result, 0) - mock_repo.get_pulls.assert_called_once() call_kwargs = mock_repo.get_pulls.call_args.kwargs - self.assertEqual(call_kwargs["state"], "open") + # Must query *all* PRs (not just open) so we can detect closed/merged + # ones and handle them without a duplicate-PR error. + self.assertEqual(call_kwargs["state"], "all") self.assertEqual(call_kwargs["head"], "DataDog:bump-rshell-v0.0.99") + def test_does_not_short_circuit_on_closed_unmerged_pr(self): + # Closed-but-not-merged PR should NOT stop the flow — the early exit is + # only for open PRs. We prove this by letting the subprocess calls + # run; the first one (git clone) fails in the test env, which means + # we've exited early only if subprocess was *never* called. + closed_pr = MagicMock(state="closed", merged=False) + self._setup_pulls([closed_pr]) + + with patch.dict(os.environ, {"GITHUB_TOKEN": "fake-token"}): + with patch.object(sys, "argv", ["bump.py", "v0.0.99"]): + with patch("bump.run") as mock_run: + # Have the helper raise on first call so we don't actually + # hit the network / disk. + mock_run.side_effect = RuntimeError("stop here") + with self.assertRaises(RuntimeError): + bump.main() + self.assertTrue( + mock_run.called, + "bump should have proceeded past the early-exit check", + ) + class TestTokenScrubbing(unittest.TestCase): def test_github_token_removed_from_environ_before_subprocess_calls(self): - # Use the "PR already exists" path because it goes just far enough to - # create the GitHub client — which is where the token should get - # scrubbed — without needing to mock clone/go/dda. - existing_pr = MagicMock() - existing_pr.html_url = "https://github.com/DataDog/datadog-agent/pull/1" + # Use the "open PR already exists" path because it goes just far + # enough to create the GitHub client — which is where the token should + # get scrubbed — without needing to mock clone/go/dda. + existing_pr = MagicMock( + state="open", + merged=False, + html_url="https://github.com/DataDog/datadog-agent/pull/1", + ) mock_repo = MagicMock() mock_repo.get_pulls.return_value = [existing_pr] _github_stub.Github.reset_mock()