From ffd87722dc21bdd7b585a4ed248f609f516f373b Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Wed, 22 Apr 2026 22:53:54 +0700 Subject: [PATCH 01/23] feat: enable log forwarding through otel collector --- .../enable_log_forwarding_action_tests.yaml | 76 +++++++++ actions/enable-log-forwarding/action.yaml | 31 ++++ .../enable-log-forwarding.py | 160 ++++++++++++++++++ .../tests/test_enable_log_forwarding.py | 88 ++++++++++ docs/changelog.md | 4 + docs/how-to/enable-log-forwarding.md | 52 ++++++ 6 files changed, 411 insertions(+) create mode 100644 .github/workflows/enable_log_forwarding_action_tests.yaml create mode 100644 actions/enable-log-forwarding/action.yaml create mode 100644 actions/enable-log-forwarding/enable-log-forwarding.py create mode 100644 actions/enable-log-forwarding/tests/test_enable_log_forwarding.py create mode 100644 docs/how-to/enable-log-forwarding.md diff --git a/.github/workflows/enable_log_forwarding_action_tests.yaml b/.github/workflows/enable_log_forwarding_action_tests.yaml new file mode 100644 index 00000000..4353a671 --- /dev/null +++ b/.github/workflows/enable_log_forwarding_action_tests.yaml @@ -0,0 +1,76 @@ +name: Enable Log Forwarding Action Tests + +on: + pull_request: + workflow_call: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + detect-changes: + runs-on: ubuntu-latest + outputs: + action: ${{ steps.filter.outputs.action }} + steps: + - uses: actions/checkout@v6 + - uses: dorny/paths-filter@v4 + id: filter + with: + filters: | + action: + - 'actions/enable-log-forwarding/**' + - '.github/workflows/enable_log_forwarding_action_tests.yaml' + + test-action: + needs: detect-changes + if: ${{ needs.detect-changes.outputs.action == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Validate Python syntax + run: python -m py_compile actions/enable-log-forwarding/enable-log-forwarding.py + + - name: Run unit tests + run: python -m unittest discover -s actions/enable-log-forwarding/tests -p 'test_*.py' -v + + smoke-test-self-hosted: + needs: detect-changes + if: ${{ needs.detect-changes.outputs.action == 'true' }} + runs-on: [self-hosted-linux-amd64-noble-edge] + env: + TEST_CONFIG_FILE: 98-enable-log-forwarding-smoke-${{ github.run_id }}-${{ github.run_attempt }}.yaml + steps: + - uses: actions/checkout@v6 + + - name: Run enable log forwarding action + uses: ./actions/enable-log-forwarding + with: + files: | + /var/log/syslog + config-file-name: ${{ env.TEST_CONFIG_FILE }} + otlp-endpoint: 127.0.0.1:4318 + + - name: Verify generated config file exists + run: | + sudo test -f /etc/otelcol/config.d/${TEST_CONFIG_FILE} + + - name: Verify generated config contains expected receiver + run: | + sudo grep -q '"filelog/github_runner_optin"' /etc/otelcol/config.d/${TEST_CONFIG_FILE} + + - name: Clean up generated config + if: always() + run: | + sudo rm -f /etc/otelcol/config.d/${TEST_CONFIG_FILE} + sudo snap restart opentelemetry-collector diff --git a/actions/enable-log-forwarding/action.yaml b/actions/enable-log-forwarding/action.yaml new file mode 100644 index 00000000..e7bd9823 --- /dev/null +++ b/actions/enable-log-forwarding/action.yaml @@ -0,0 +1,31 @@ +name: Enable log forwarding +description: Opt in to forward selected log files from a self-hosted GitHub runner to Loki. + +inputs: + files: + description: | + Newline or comma separated list of file paths or glob patterns to forward. + Example: /var/log/chrony/*.log + required: true + otlp-endpoint: + description: | + Optional gRPC endpoint for upstream OpenTelemetry Collector logs export. + When not set, the action falls back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT. + Example: otel-gateway.internal:4318 + required: false + default: "" + config-file-name: + description: File name for the generated collector fragment. + required: false + default: 90-github-runner-log-forwarding.yaml + +runs: + using: composite + steps: + - name: Configure collector for opt-in file log forwarding + shell: python3 {0} + env: + INPUT_FILES: ${{ inputs.files }} + INPUT_OTLP_ENDPOINT: ${{ inputs.otlp-endpoint }} + INPUT_CONFIG_FILE_NAME: ${{ inputs.config-file-name }} + run: ${{ github.action_path }}/enable-log-forwarding.py diff --git a/actions/enable-log-forwarding/enable-log-forwarding.py b/actions/enable-log-forwarding/enable-log-forwarding.py new file mode 100644 index 00000000..0f571bdc --- /dev/null +++ b/actions/enable-log-forwarding/enable-log-forwarding.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +CONFIG_DIR = "/etc/otelcol/config.d" +EXPORTER_NAME = "otlp_grpc" + + +def run_as_root(*args): + if os.geteuid() == 0: # if running as root + return subprocess.run(args, capture_output=True) + if shutil.which("sudo"): # if sudo is available + return subprocess.run(["sudo", *args], capture_output=True) + print("This action requires root privileges to update collector config.", file=sys.stderr) + sys.exit(1) + + +def parse_files_into_list(files_input): + entries = [] + # Split on commas or newlines, and strip whitespace. Ignore empty entries. + for item in re.split(r"[,\n]", files_input): + stripped = item.strip() + if stripped: + entries.append(stripped) + return entries + + +def resolve_endpoint(): + # If INPUT_OTLP_ENDPOINT is not set, fall back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT + for env_var in ( + "INPUT_OTLP_ENDPOINT", + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", + ): + val = os.environ.get(env_var, "").strip() + if val: + return val + return "" + + +def check_exporter_exists(): + # Check for " otlp_grpc:" exporter definition + pattern = f"^ {re.escape(EXPORTER_NAME)}:[ \t]*$" + if run_as_root("test", "-d", CONFIG_DIR).returncode != 0: + return False + # Search for the exporter definition in all files under CONFIG_DIR + if run_as_root("grep", "-RqsE", pattern, CONFIG_DIR).returncode == 0: + return True + return False + + +def build_config(files, resolved_endpoint, exporter_already_exists): + attrs = [ + ("github.repository", os.environ.get("GITHUB_REPOSITORY", "unknown")), + ("github.runner.name", os.environ.get("RUNNER_NAME", "unknown")), + ("github.workflow", os.environ.get("GITHUB_WORKFLOW", "unknown")), + ("github.job.name", os.environ.get("GITHUB_JOB", "unknown")), + ("github.job.id", os.environ.get("GITHUB_RUN_ID", "unknown")), + ("github.run.attempt", os.environ.get("GITHUB_RUN_ATTEMPT", "unknown")), + ] + config = { + "receivers": { + "filelog/github_runner_optin": { + "include": files, + "start_at": "end", + } + }, + "processors": { + "resource/github_runner_optin": { + "attributes": [ + {"key": key, "value": value, "action": "upsert"} + for key, value in attrs + ] + } + }, + "service": { + "pipelines": { + "logs/github_runner_optin": { + "receivers": ["filelog/github_runner_optin"], + "processors": ["resource/github_runner_optin", "batch"], + "exporters": [EXPORTER_NAME], + } + } + }, + } + if not exporter_already_exists and resolved_endpoint: + config["exporters"] = { + EXPORTER_NAME: {"endpoint": resolved_endpoint} + } + return json.dumps(config, indent=2) + "\n" + + +def main(): + files_input = os.environ.get("INPUT_FILES", "").strip() + if not files_input: + print("Input 'files' cannot be empty.", file=sys.stderr) + sys.exit(1) + + config_file_name = os.environ.get("INPUT_CONFIG_FILE_NAME", "90-github-runner-log-forwarding.yaml").strip() + if "/" in config_file_name: + print("Input 'config-file-name' must not include directory separators.", file=sys.stderr) + sys.exit(1) + + config_path = os.path.join(CONFIG_DIR, config_file_name) + + if shutil.which("snap") is None: + print("Required command is missing: snap", file=sys.stderr) + sys.exit(1) + + if subprocess.run(["snap", "list", "opentelemetry-collector"], capture_output=True).returncode != 0: + print("opentelemetry-collector snap is not installed on this runner.", file=sys.stderr) + sys.exit(1) + + files = parse_files_into_list(files_input) + if not files: + print("Input 'files' must contain at least one path or glob.", file=sys.stderr) + sys.exit(1) + + resolved_endpoint = resolve_endpoint() + exporter_already_exists = check_exporter_exists() + + if not exporter_already_exists and not resolved_endpoint: + print( + f"Exporter '{EXPORTER_NAME}' was not found in scanned collector config directories and no OTLP endpoint was provided.", + file=sys.stderr, + ) + print( + "Set input 'otlp-endpoint', or expose ACTION_OTEL_EXPORTER_OTLP_ENDPOINT to this workflow.", + file=sys.stderr, + ) + print( + f"The generated pipeline will still reference '{EXPORTER_NAME}'. Collector restart may fail if that exporter is undefined.", + file=sys.stderr, + ) + + config_content = build_config(files, resolved_endpoint, exporter_already_exists) + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as tmp: + tmp.write(config_content) + tmp_path = tmp.name + + try: + run_as_root("mkdir", "-p", CONFIG_DIR) # create if missing, do nothing if exists + run_as_root("install", "-m", "0644", tmp_path, config_path) # owner read/write and group/other read permissions + finally: + os.unlink(tmp_path) + + print(f"Wrote log-forwarding collector config to: {config_path}") + + run_as_root("snap", "restart", "opentelemetry-collector") + print("Restarted opentelemetry-collector to apply log-forwarding config.") + + +if __name__ == "__main__": + main() diff --git a/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py b/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py new file mode 100644 index 00000000..47ae5fb7 --- /dev/null +++ b/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py @@ -0,0 +1,88 @@ +import importlib.util +import json +import pathlib +import types +import unittest +from unittest import mock + + +MODULE_PATH = pathlib.Path(__file__).resolve().parent.parent / "enable-log-forwarding.py" + + +def load_module(path: pathlib.Path): + spec = importlib.util.spec_from_file_location("enable_log_forwarding", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load module from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +MODULE = load_module(MODULE_PATH) + + +class TestEnableLogForwarding(unittest.TestCase): + def test_parse_files_into_list(self): + files_input = " /var/log/a.log,\n/var/log/b.log ,, /var/log/c*.log\n" + parsed = MODULE.parse_files_into_list(files_input) + self.assertEqual(parsed, ["/var/log/a.log", "/var/log/b.log", "/var/log/c*.log"]) + + def test_resolve_endpoint_prefers_input(self): + with mock.patch.dict( + MODULE.os.environ, + { + "INPUT_OTLP_ENDPOINT": "input-endpoint:4318", + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT": "system-endpoint:4318", + }, + clear=False, + ): + resolved = MODULE.resolve_endpoint() + + self.assertEqual(resolved, "input-endpoint:4318") + + def test_resolve_endpoint_falls_back_to_action_env(self): + with mock.patch.dict( + MODULE.os.environ, + { + "INPUT_OTLP_ENDPOINT": "", + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT": "system-endpoint:4318", + }, + clear=False, + ): + resolved = MODULE.resolve_endpoint() + + self.assertEqual(resolved, "system-endpoint:4318") + + def test_check_exporter_exists_true(self): + # First call checks directory exists, second call checks grep match. + calls = [types.SimpleNamespace(returncode=0), types.SimpleNamespace(returncode=0)] + with mock.patch.object(MODULE, "run_as_root", side_effect=calls): + self.assertTrue(MODULE.check_exporter_exists()) + + def test_build_config_adds_exporter_when_missing(self): + env = { + "GITHUB_REPOSITORY": "canonical/github-runner-operators", + "RUNNER_NAME": "runner-1", + "GITHUB_WORKFLOW": "CI", + "GITHUB_JOB": "test", + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "1", + } + with mock.patch.dict(MODULE.os.environ, env, clear=False): + raw = MODULE.build_config(["/var/log/syslog"], "otel:4318", False) + + config = json.loads(raw) + self.assertEqual(config["receivers"]["filelog/github_runner_optin"]["include"], ["/var/log/syslog"]) + self.assertEqual(config["service"]["pipelines"]["logs/github_runner_optin"]["exporters"], [MODULE.EXPORTER_NAME]) + self.assertEqual(config["exporters"][MODULE.EXPORTER_NAME]["endpoint"], "otel:4318") + + def test_build_config_reuses_existing_exporter(self): + with mock.patch.dict(MODULE.os.environ, {}, clear=False): + raw = MODULE.build_config(["/var/log/syslog"], "otel:4318", True) + + config = json.loads(raw) + self.assertNotIn("exporters", config) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/changelog.md b/docs/changelog.md index 4675c549..8dcf046f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ This changelog documents user-relevant changes to the Planner charm and Webhook gateway charm. +## 2026-04-22 + +- add action to allow workflow authors to opt in to forwarding specific log files from self-hosted GitHub runners to Loki through the OpenTelemetry Collector snap. + ## 2026-04-13 - add 5xx error logging to planner routes. diff --git a/docs/how-to/enable-log-forwarding.md b/docs/how-to/enable-log-forwarding.md new file mode 100644 index 00000000..ae77db20 --- /dev/null +++ b/docs/how-to/enable-log-forwarding.md @@ -0,0 +1,52 @@ +# Enable log forwarding + +This action allows workflow authors to opt in to forwarding specific log files from self-hosted GitHub runners to Loki through the OpenTelemetry Collector snap. + +By default, nothing is forwarded. Log forwarding starts only when this action is used in a workflow. + +## Inputs + +- `files` (required): newline or comma separated file paths or glob patterns. +- `config-file-name` (optional, default `90-github-runner-log-forwarding.yaml`): generated config file name. +- `otlp-endpoint` (optional): OTLP/HTTP endpoint used to create the exporter when one is not already configured. + +When `otlp-endpoint` is not set, the action falls back to `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`, then `OTEL_EXPORTER_OTLP_ENDPOINT` from the workflow environment. + +## Usage + +```yaml +jobs: + chrony-testing: + runs-on: ubuntu-latest + steps: + - uses: canonical/github-runner-operators/actions/enable-log-forwarding@main + with: + files: | + /var/log/chrony/*.log + /var/log/syslog + - run: ./run-tests.sh +``` + +Pin to a release tag or commit SHA in production workflows. + +## Loki query hints + +The action adds GitHub context as resource attributes on forwarded logs: + +- `github.job.id` +- `github.job.name` +- `github.repository` +- `github.runner.name` +- `github.workflow` +- `github.run.attempt` + +Example Loki query by workflow run id: + +```logql +{github_job_id="123456789"} +``` + +## Notes + +- This action requires root privileges to write collector config. +- The `opentelemetry-collector` snap must be installed on the runner. From 14514f26381f7bf0c817dfb5ae313fbdcb2d65b3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 18 Apr 2026 03:15:49 +0000 Subject: [PATCH 02/23] chore(deps): update dependency packaging to v26.1 (#180) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 68e90133..37c3122b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -22,7 +22,7 @@ sphinx-ubuntu-images==0.1.0 sphinx-youtube-links==0.1.0 # Other dependencies -packaging==26.0 +packaging==26.1 sphinxcontrib-svg2pdfconverter[CairoSVG]==2.1.0 sphinx-last-updated-by-git==0.3.8 sphinx-sitemap==2.9.0 From 77f12cf3d7df5b86bdd028f86fdec4b83ff2da5d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 18 Apr 2026 07:39:34 +0000 Subject: [PATCH 03/23] chore(deps): update dependency sphinx-ubuntu-images to v0.2.0 (#181) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 37c3122b..dbec93e9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -18,7 +18,7 @@ sphinx-filtered-toctree==0.1.0 sphinx-related-links==0.1.2 sphinx-roles==0.1.0 sphinx-terminal==1.0.3 -sphinx-ubuntu-images==0.1.0 +sphinx-ubuntu-images==0.2.0 sphinx-youtube-links==0.1.0 # Other dependencies From f41fd562e84036ac82465206591d1a1da7da8d48 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 19 Apr 2026 12:18:53 +0000 Subject: [PATCH 04/23] chore(deps): replace astral-sh/setup-uv action with astral-sh/setup-uv v8 (#182) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/charms_lint_and_unit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/charms_lint_and_unit.yaml b/.github/workflows/charms_lint_and_unit.yaml index 30852287..da5a8daa 100644 --- a/.github/workflows/charms_lint_and_unit.yaml +++ b/.github/workflows/charms_lint_and_unit.yaml @@ -44,7 +44,7 @@ jobs: python-version: "3.12" - name: Set up uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@v8 - name: Install tox run: uv tool install tox --with tox-uv From 11631ad2bd4c2bcdec17d9b9b638cb496deecf94 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 19 Apr 2026 15:28:39 +0000 Subject: [PATCH 05/23] fix(deps): update module github.com/jackc/pgx/v5 to v5.9.2 (#183) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 03dfb225..ca0885fe 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.0 require ( github.com/golang-migrate/migrate/v4 v4.19.1 github.com/google/go-github/v82 v82.0.0 - github.com/jackc/pgx/v5 v5.9.1 + github.com/jackc/pgx/v5 v5.9.2 github.com/prometheus/client_golang v1.23.2 github.com/rabbitmq/amqp091-go v1.10.0 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 719eb7a2..81eb2ecc 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7Ulw github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= From b2cff4d4a07ea214f7e4443a315c43c04048df83 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 22 Apr 2026 11:58:45 +0200 Subject: [PATCH 06/23] fix(dashboard): remove dead override hiding job queue time series (#184) * fix(dashboard): remove dead override hiding job queue time series The "Job queue time" panel had a leftover hideSeriesFrom override that excluded every series except one specific named expression. Combined with the panel's current query (which already aggregates with sum by (le)), the override hid all data, leaving an empty chart. The override is obsolete now that the query collapses every label except le into a single combined histogram, so removing it restores visualisation without any other change. * ci: pin astral-sh/setup-uv to v8.1.0 The astral-sh/setup-uv repository does not publish a floating v8 major tag (only v8.0.0 and v8.1.0 specific tags exist), so referencing @v8 fails to resolve and breaks the workflow on every PR. --- .github/workflows/charms_lint_and_unit.yaml | 2 +- .../cos_custom/grafana_dashboards/go.json | 27 +------------------ 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/.github/workflows/charms_lint_and_unit.yaml b/.github/workflows/charms_lint_and_unit.yaml index da5a8daa..31bf04b6 100644 --- a/.github/workflows/charms_lint_and_unit.yaml +++ b/.github/workflows/charms_lint_and_unit.yaml @@ -44,7 +44,7 @@ jobs: python-version: "3.12" - name: Set up uv - uses: astral-sh/setup-uv@v8 + uses: astral-sh/setup-uv@v8.1.0 - name: Install tox run: uv tool install tox --with tox-uv diff --git a/charms/planner-operator/cos_custom/grafana_dashboards/go.json b/charms/planner-operator/cos_custom/grafana_dashboards/go.json index 3da1d40a..f8b9878d 100644 --- a/charms/planner-operator/cos_custom/grafana_dashboards/go.json +++ b/charms/planner-operator/cos_custom/grafana_dashboards/go.json @@ -590,32 +590,7 @@ }, "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "histogram_quantile(0.95, \nsum by (le) (\n rate(github_runner_planner_webhook_job_waiting_seconds_bucket[5m])\n )\n)" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, From 564ff2ca4afd953e30488244517b464dcfb1e8b6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 14:16:39 +0000 Subject: [PATCH 07/23] chore: update Copilot collections to v0.11.0 (#172) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Christopher Bartz --- .copilot-collections.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.copilot-collections.yaml b/.copilot-collections.yaml index bc72a544..1fb2d0d1 100644 --- a/.copilot-collections.yaml +++ b/.copilot-collections.yaml @@ -1,5 +1,5 @@ copilot: - version: "v0.7.0" + version: "v0.11.0" collections: - charm-python - pfe-charms From 7b44fd01f453c19b0ec69585eabbc8bd80196154 Mon Sep 17 00:00:00 2001 From: Erin Conley Date: Wed, 22 Apr 2026 11:26:25 -0400 Subject: [PATCH 08/23] chore(docs): update contributing guidelines (charmkeeper) (#170) * chore(docs): update contributing guidelines (charmkeeper) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix(docs): build errors * chore: clarify PR checklist and remove unnecessary item * chore(docs): revert changes in CONTRIBUTING.md * fix(docs): whoops we're ignoring the changelog * fix: update pr checklist to incorporate previous items, remove duplicate items --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/pull_request_template.md | 30 +++++++++++---- CONTRIBUTING.md | 54 ++++++++++++++++++--------- docs/changelog.md | 6 +++ docs/how-to/contribute.rst | 63 ++++++++++++++++++++++++++++++++ docs/how-to/index.rst | 7 ++++ 5 files changed, 135 insertions(+), 25 deletions(-) create mode 100644 docs/how-to/contribute.rst diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c047a37b..2b989110 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,15 +1,29 @@ -### Overview +#### What this PR does - +#### Why we need it -### Rationale - - - -### Checklist +#### Checklist - [ ] Changes comply with the project's coding standards and guidelines (see CONTRIBUTING.md and STYLE.md) - [ ] `CONTRIBUTING.md` has been updated upon changes to the contribution/development process (e.g. changes to the way tests are run) - [ ] Technical author has been assigned to review the PR in case of documentation changes (usually *.md files) +- [ ] I updated `docs/changelog.md` with user-relevant changes +- [ ] I used AI to assist with preparing this PR +- [ ] I added or updated tests as needed (unit and integration) +- [ ] **If integration test modules are used:** I updated the workflow configuration + (e.g., in `.github/workflows/integration_tests.yaml`, ensure the `modules` list is correct) +- [ ] **If this PR involves a Grafana dashboard:** I added a screenshot of the dashboard +- [ ] **If this PR involves Terraform:** `terraform fmt` passes and `tflint` reports no errors +- [ ] **If this PR involves Rockcraft:** I updated the version + + - \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c4d853d..66d24e34 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,9 +1,9 @@ -# Contribute - -## Overview +# Contributing This document explains the processes and practices recommended for contributing enhancements to the codebase. +## Overview + - Generally, before developing enhancements to this code base, you should consider [opening an issue](https://github.com/canonical/github-runner-operator/issues) explaining your use case. - If you would like to chat with us about your use-cases or proposed implementation, you can reach us at [Canonical Charm Development Matrix public channel](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) or [Discourse](https://discourse.charmhub.io/). - All enhancements require review before being merged. Code review typically examines @@ -15,6 +15,14 @@ This document explains the processes and practices recommended for contributing When contributing, you must abide by the [Ubuntu Code of Conduct](https://ubuntu.com/community/ethos/code-of-conduct). +## Changelog + +Please ensure that any new feature, fix, or significant change is documented by +adding an entry to the [CHANGELOG.md](docs/changelog.md) file. Use the date of the +contribution as the header for new entries. + +To learn more about changelog best practices, visit [Keep a Changelog](https://keepachangelog.com/). + ## Submissions If you want to address an issue or a bug in this project, @@ -32,21 +40,31 @@ also, reference the issue or bug number when you submit the changes. Your changes will be reviewed in due time; if approved, they will be eventually merged. -### Describing pull requests +### AI -To be properly considered, reviewed and merged, -your pull request must provide the following details: +You are free to use any tools you want while preparing your contribution, including +AI, provided that you do so lawfully and ethically. -- **Title**: Summarize the change in a short, descriptive title. +Avoid using AI to complete issues tagged with the "good first issues" label. The +purpose of these issues is to provide newcomers with opportunities to contribute +to our projects and gain coding skills. Using AI to complete these tasks +undermines their purpose. -- **Overview**: Describe the problem that your pull request solves. - Mention any new features, bug fixes or refactoring. +We have created instructions and tools that you can provide AI while preparing your contribution: [`copilot-collections`](https://github.com/canonical/copilot-collections) -- **Rationale**: Explain why the change is needed. +While it isn't necessary to use `copilot-collections` while preparing your +contribution, these files contain details about our quality standards and +practices that will help the AI avoid common pitfalls when interacting with +our projects. By using these tools, you can avoid longer review times and nitpicks. -- **Checklist**: Complete the following items: +If you choose to use AI, please disclose this information to us by indicating +AI usage in the PR description (for instance, marking the checklist item about +AI usage). You don't need to go into explicit details about how and where you used AI. - - The PR is tagged with appropriate label (`urgent`, `trivial`, `senior-review-required`, `documentation`). +Avoid submitting contributions that you don't fully understand. +You are responsible for the entire contribution, including the AI-assisted portions. +You must be willing to engage in discussion and respond to any questions, comments, +or suggestions we may have. ### Signing commits @@ -54,9 +72,14 @@ To improve contribution tracking, we use the [Canonical contributor license agreement](https://assets.ubuntu.com/v1/ff2478d1-Canonical-HA-CLA-ANY-I_v1.2.pdf) (CLA) as a legal sign-off, and we require all commits to have verified signatures. -### Canonical contributor agreement +#### Canonical contributor agreement + +Canonical welcomes contributions to the GitHub runner Operator. Please check out our +[contributor agreement](https://ubuntu.com/legal/contributors) if you're interested in contributing to the solution. -Canonical welcomes contributions to this repository. Please check out our [contributor agreement](https://ubuntu.com/legal/contributors) if you’re interested in contributing to the solution. +The CLA sign-off is simple line at the +end of the commit message certifying that you wrote it +or have the right to commit it as an open-source contribution. #### Verified signatures on commits @@ -87,7 +110,6 @@ We like to follow idomatic Go practices and community standards when writing Go We have added an instruction file `go.instructions.md` in `.github/instructions.md` that is used by GitHub Copilot to help you write code that follows these practices. We have added a [Style Guide](./STYLE.md) that you can refer to for more details. - ### Test This project uses standard Go testing tools for unit tests and integration tests. @@ -169,8 +191,6 @@ Higher complexity leads to code that is harder to read, understand, test and mai There are exceptions where higher complexity is justified (e.g., validation, initialization), but those should require explicit justification using `nolint` directives. - - ### Charm development The charm uses the [12 factor app pattern](https://canonical-12-factor-app-support.readthedocs-hosted.com/latest/). diff --git a/docs/changelog.md b/docs/changelog.md index 8dcf046f..fd7dd68c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,7 +1,13 @@ +(changelog)= + # Changelog This changelog documents user-relevant changes to the Planner charm and Webhook gateway charm. +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +Each revision is versioned by the date of the revision. + ## 2026-04-22 - add action to allow workflow authors to opt in to forwarding specific log files from self-hosted GitHub runners to Loki through the OpenTelemetry Collector snap. diff --git a/docs/how-to/contribute.rst b/docs/how-to/contribute.rst new file mode 100644 index 00000000..eec9fd2c --- /dev/null +++ b/docs/how-to/contribute.rst @@ -0,0 +1,63 @@ +.. meta:: + :description: Familiarize yourself with contributing to the GitHub runner charms documentation. + +.. _how_to_contribute: + +How to contribute +================= + +.. note:: + + See `CONTRIBUTING.md `_ + for information on contributing to the source code. + +Our documentation is hosted on `Read the Docs `_ to enable collaboration. +Please use the links on each documentation page to either +directly change something you see that's wrong, ask a question, or make a suggestion +about a potential change. + +Our documentation is also available alongside the +`source code on GitHub `_. +You may open a pull request with your documentation changes, or you can +`file a bug `_ +to provide constructive feedback or suggestions. + +AI usage +-------- + +You are free to use any tools you want while preparing your contribution, including +AI, provided that you do so lawfully and ethically. + +Avoid using AI to complete +`Canonical Open Documentation Academy issues `_. +The purpose of these issues is to provide newcomers with opportunities to +contribute to our projects and gain documentation skills. Using AI to +complete these tasks undermines their purpose. + +If you use AI to help with your PRs, be mindful. Avoid submitting contributions +with entirely AI-generated documentation. The human aspect of documentation is +important to us, and that includes tone, syntax, perspectives, and the +occasional typo. + +Some examples of valid AI assistance includes: + +* Checking for spelling or grammar errors +* Drafting plans or outlines +* Checking that your contribution aligns with the Canonical style guide + +We have created instructions and tools that you can provide AI while preparing +your contribution in `copilot-collections `_. +While it isn't necessary to use ``copilot-collections`` while preparing your +contribution, these files contain details about our documentation standards and +practices that will help the AI avoid common pitfalls when interacting with our +projects. By using these tools, you can avoid longer review times and nitpicks. + +If you choose to use AI, please disclose this information to us by indicating +AI usage in the PR description (for instance, marking the checklist item about +AI usage). You don't need to go into explicit details about how and where you used AI. + +Avoid submitting contributions that you don't fully understand. +You are responsible for the entire contribution, including the AI-assisted portions. +You must be willing to engage in discussion and respond to any questions, comments, +or suggestions we may have. + diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst index 8ca5a71b..47992476 100644 --- a/docs/how-to/index.rst +++ b/docs/how-to/index.rst @@ -1,2 +1,9 @@ How-to guides ============= + +The following guides cover key processes and common tasks for managing and using the GitHub runner charms. + +.. toctree:: + :maxdepth: 1 + + Contribute From 1bdb99d06fdb0b5b3468ea85b6da048b350a2bb6 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Wed, 22 Apr 2026 23:02:22 +0700 Subject: [PATCH 09/23] fix: run shell as bash --- actions/enable-log-forwarding/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/enable-log-forwarding/action.yaml b/actions/enable-log-forwarding/action.yaml index e7bd9823..c467104c 100644 --- a/actions/enable-log-forwarding/action.yaml +++ b/actions/enable-log-forwarding/action.yaml @@ -23,9 +23,9 @@ runs: using: composite steps: - name: Configure collector for opt-in file log forwarding - shell: python3 {0} + shell: bash env: INPUT_FILES: ${{ inputs.files }} INPUT_OTLP_ENDPOINT: ${{ inputs.otlp-endpoint }} INPUT_CONFIG_FILE_NAME: ${{ inputs.config-file-name }} - run: ${{ github.action_path }}/enable-log-forwarding.py + run: python3 "${{ github.action_path }}/enable-log-forwarding.py" From 32eb82f3ce306c6ba4b8c688531439f6c4bae466 Mon Sep 17 00:00:00 2001 From: florentianayuwono <76247368+florentianayuwono@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:21:17 +0700 Subject: [PATCH 10/23] Update docs/how-to/enable-log-forwarding.md Co-authored-by: Erin Conley --- docs/how-to/enable-log-forwarding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/enable-log-forwarding.md b/docs/how-to/enable-log-forwarding.md index ae77db20..ff28f012 100644 --- a/docs/how-to/enable-log-forwarding.md +++ b/docs/how-to/enable-log-forwarding.md @@ -1,6 +1,6 @@ # Enable log forwarding -This action allows workflow authors to opt in to forwarding specific log files from self-hosted GitHub runners to Loki through the OpenTelemetry Collector snap. +The `enable-log-forwarding` action allows workflow authors to opt in to forwarding specific log files from self-hosted GitHub runners to Loki through the OpenTelemetry Collector snap. By default, nothing is forwarded. Log forwarding starts only when this action is used in a workflow. From f7f41476d6007f562db132c6cf01025e36c147ac Mon Sep 17 00:00:00 2001 From: florentianayuwono <76247368+florentianayuwono@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:27:58 +0700 Subject: [PATCH 11/23] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Erin Conley --- docs/how-to/enable-log-forwarding.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/how-to/enable-log-forwarding.md b/docs/how-to/enable-log-forwarding.md index ff28f012..3c97ec58 100644 --- a/docs/how-to/enable-log-forwarding.md +++ b/docs/how-to/enable-log-forwarding.md @@ -4,15 +4,15 @@ The `enable-log-forwarding` action allows workflow authors to opt in to forwardi By default, nothing is forwarded. Log forwarding starts only when this action is used in a workflow. -## Inputs +## Provide inputs - `files` (required): newline or comma separated file paths or glob patterns. - `config-file-name` (optional, default `90-github-runner-log-forwarding.yaml`): generated config file name. - `otlp-endpoint` (optional): OTLP/HTTP endpoint used to create the exporter when one is not already configured. -When `otlp-endpoint` is not set, the action falls back to `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`, then `OTEL_EXPORTER_OTLP_ENDPOINT` from the workflow environment. +When `otlp-endpoint` is not set, the action falls back to `ACTION_OTEL_EXPORTER_OTLP_ENDPOINT` from the workflow environment. -## Usage +## Use the action ```yaml jobs: @@ -29,7 +29,7 @@ jobs: Pin to a release tag or commit SHA in production workflows. -## Loki query hints +## Examine Loki queries The action adds GitHub context as resource attributes on forwarded logs: From 0539529bac14079e2dd7a5c326bd0c55c76cb6a6 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 23 Apr 2026 10:28:33 +0700 Subject: [PATCH 12/23] fix docs --- docs/how-to/enable-log-forwarding.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/enable-log-forwarding.md b/docs/how-to/enable-log-forwarding.md index 3c97ec58..d5aded4f 100644 --- a/docs/how-to/enable-log-forwarding.md +++ b/docs/how-to/enable-log-forwarding.md @@ -1,4 +1,4 @@ -# Enable log forwarding +# How to enable log forwarding The `enable-log-forwarding` action allows workflow authors to opt in to forwarding specific log files from self-hosted GitHub runners to Loki through the OpenTelemetry Collector snap. From 16a1ff704477dfc0defdfc62f54631ea7ce1907c Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 23 Apr 2026 10:28:44 +0700 Subject: [PATCH 13/23] add index --- docs/how-to/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/how-to/index.rst b/docs/how-to/index.rst index 47992476..27cca196 100644 --- a/docs/how-to/index.rst +++ b/docs/how-to/index.rst @@ -7,3 +7,4 @@ The following guides cover key processes and common tasks for managing and using :maxdepth: 1 Contribute + Enable log forwarding From 9934f4e387c04ad49166156acb89392289e8c33a Mon Sep 17 00:00:00 2001 From: florentianayuwono <76247368+florentianayuwono@users.noreply.github.com> Date: Thu, 23 Apr 2026 20:02:08 +0700 Subject: [PATCH 14/23] Update actions/enable-log-forwarding/enable-log-forwarding.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- actions/enable-log-forwarding/enable-log-forwarding.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/enable-log-forwarding/enable-log-forwarding.py b/actions/enable-log-forwarding/enable-log-forwarding.py index 0f571bdc..7d11d0c6 100644 --- a/actions/enable-log-forwarding/enable-log-forwarding.py +++ b/actions/enable-log-forwarding/enable-log-forwarding.py @@ -59,8 +59,8 @@ def build_config(files, resolved_endpoint, exporter_already_exists): ("github.repository", os.environ.get("GITHUB_REPOSITORY", "unknown")), ("github.runner.name", os.environ.get("RUNNER_NAME", "unknown")), ("github.workflow", os.environ.get("GITHUB_WORKFLOW", "unknown")), - ("github.job.name", os.environ.get("GITHUB_JOB", "unknown")), - ("github.job.id", os.environ.get("GITHUB_RUN_ID", "unknown")), + ("github.job.id", os.environ.get("GITHUB_JOB", "unknown")), + ("github.run.id", os.environ.get("GITHUB_RUN_ID", "unknown")), ("github.run.attempt", os.environ.get("GITHUB_RUN_ATTEMPT", "unknown")), ] config = { From 83374204c51db4666af2f1ae5890fbd2646c6f7b Mon Sep 17 00:00:00 2001 From: florentianayuwono <76247368+florentianayuwono@users.noreply.github.com> Date: Thu, 23 Apr 2026 20:03:59 +0700 Subject: [PATCH 15/23] Update actions/enable-log-forwarding/enable-log-forwarding.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- actions/enable-log-forwarding/enable-log-forwarding.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/actions/enable-log-forwarding/enable-log-forwarding.py b/actions/enable-log-forwarding/enable-log-forwarding.py index 7d11d0c6..c8354a7e 100644 --- a/actions/enable-log-forwarding/enable-log-forwarding.py +++ b/actions/enable-log-forwarding/enable-log-forwarding.py @@ -133,10 +133,7 @@ def main(): "Set input 'otlp-endpoint', or expose ACTION_OTEL_EXPORTER_OTLP_ENDPOINT to this workflow.", file=sys.stderr, ) - print( - f"The generated pipeline will still reference '{EXPORTER_NAME}'. Collector restart may fail if that exporter is undefined.", - file=sys.stderr, - ) + sys.exit(1) config_content = build_config(files, resolved_endpoint, exporter_already_exists) From 3f3fd599aa277e60441dcb2cbf19a4b404156e6f Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Fri, 24 Apr 2026 01:04:17 +0700 Subject: [PATCH 16/23] address code reviews Co-authored-by: Copilot --- .../enable_log_forwarding_action_tests.yaml | 22 +- .../enable-log-forwarding.py | 157 ------------ .../action.yaml | 6 +- .../enable_log_forwarding.py | 229 ++++++++++++++++++ .../tests/test_enable_log_forwarding.py | 45 +++- docs/how-to/enable-log-forwarding.md | 29 ++- tox.ini | 28 ++- 7 files changed, 320 insertions(+), 196 deletions(-) delete mode 100644 actions/enable-log-forwarding/enable-log-forwarding.py rename actions/{enable-log-forwarding => enable_log_forwarding}/action.yaml (82%) create mode 100644 actions/enable_log_forwarding/enable_log_forwarding.py rename actions/{enable-log-forwarding => enable_log_forwarding}/tests/test_enable_log_forwarding.py (59%) diff --git a/.github/workflows/enable_log_forwarding_action_tests.yaml b/.github/workflows/enable_log_forwarding_action_tests.yaml index 4353a671..4761e4fe 100644 --- a/.github/workflows/enable_log_forwarding_action_tests.yaml +++ b/.github/workflows/enable_log_forwarding_action_tests.yaml @@ -23,7 +23,7 @@ jobs: with: filters: | action: - - 'actions/enable-log-forwarding/**' + - 'actions/enable_log_forwarding/**' - '.github/workflows/enable_log_forwarding_action_tests.yaml' test-action: @@ -38,15 +38,18 @@ jobs: with: python-version: "3.12" - - name: Validate Python syntax - run: python -m py_compile actions/enable-log-forwarding/enable-log-forwarding.py + - name: Set up uv + uses: astral-sh/setup-uv@v8.1.0 - - name: Run unit tests - run: python -m unittest discover -s actions/enable-log-forwarding/tests -p 'test_*.py' -v + - name: Install tox + run: uv tool install tox --with tox-uv + + - name: Run lint, static checks, and unit tests + run: tox -e actions-lint,actions-static,actions-unit smoke-test-self-hosted: needs: detect-changes - if: ${{ needs.detect-changes.outputs.action == 'true' }} + if: ${{ needs.detect-changes.outputs.action == 'true' && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'self-hosted-smoke')) }} runs-on: [self-hosted-linux-amd64-noble-edge] env: TEST_CONFIG_FILE: 98-enable-log-forwarding-smoke-${{ github.run_id }}-${{ github.run_attempt }}.yaml @@ -54,7 +57,7 @@ jobs: - uses: actions/checkout@v6 - name: Run enable log forwarding action - uses: ./actions/enable-log-forwarding + uses: ./actions/enable_log_forwarding with: files: | /var/log/syslog @@ -69,8 +72,3 @@ jobs: run: | sudo grep -q '"filelog/github_runner_optin"' /etc/otelcol/config.d/${TEST_CONFIG_FILE} - - name: Clean up generated config - if: always() - run: | - sudo rm -f /etc/otelcol/config.d/${TEST_CONFIG_FILE} - sudo snap restart opentelemetry-collector diff --git a/actions/enable-log-forwarding/enable-log-forwarding.py b/actions/enable-log-forwarding/enable-log-forwarding.py deleted file mode 100644 index c8354a7e..00000000 --- a/actions/enable-log-forwarding/enable-log-forwarding.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 - -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile - -CONFIG_DIR = "/etc/otelcol/config.d" -EXPORTER_NAME = "otlp_grpc" - - -def run_as_root(*args): - if os.geteuid() == 0: # if running as root - return subprocess.run(args, capture_output=True) - if shutil.which("sudo"): # if sudo is available - return subprocess.run(["sudo", *args], capture_output=True) - print("This action requires root privileges to update collector config.", file=sys.stderr) - sys.exit(1) - - -def parse_files_into_list(files_input): - entries = [] - # Split on commas or newlines, and strip whitespace. Ignore empty entries. - for item in re.split(r"[,\n]", files_input): - stripped = item.strip() - if stripped: - entries.append(stripped) - return entries - - -def resolve_endpoint(): - # If INPUT_OTLP_ENDPOINT is not set, fall back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT - for env_var in ( - "INPUT_OTLP_ENDPOINT", - "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", - ): - val = os.environ.get(env_var, "").strip() - if val: - return val - return "" - - -def check_exporter_exists(): - # Check for " otlp_grpc:" exporter definition - pattern = f"^ {re.escape(EXPORTER_NAME)}:[ \t]*$" - if run_as_root("test", "-d", CONFIG_DIR).returncode != 0: - return False - # Search for the exporter definition in all files under CONFIG_DIR - if run_as_root("grep", "-RqsE", pattern, CONFIG_DIR).returncode == 0: - return True - return False - - -def build_config(files, resolved_endpoint, exporter_already_exists): - attrs = [ - ("github.repository", os.environ.get("GITHUB_REPOSITORY", "unknown")), - ("github.runner.name", os.environ.get("RUNNER_NAME", "unknown")), - ("github.workflow", os.environ.get("GITHUB_WORKFLOW", "unknown")), - ("github.job.id", os.environ.get("GITHUB_JOB", "unknown")), - ("github.run.id", os.environ.get("GITHUB_RUN_ID", "unknown")), - ("github.run.attempt", os.environ.get("GITHUB_RUN_ATTEMPT", "unknown")), - ] - config = { - "receivers": { - "filelog/github_runner_optin": { - "include": files, - "start_at": "end", - } - }, - "processors": { - "resource/github_runner_optin": { - "attributes": [ - {"key": key, "value": value, "action": "upsert"} - for key, value in attrs - ] - } - }, - "service": { - "pipelines": { - "logs/github_runner_optin": { - "receivers": ["filelog/github_runner_optin"], - "processors": ["resource/github_runner_optin", "batch"], - "exporters": [EXPORTER_NAME], - } - } - }, - } - if not exporter_already_exists and resolved_endpoint: - config["exporters"] = { - EXPORTER_NAME: {"endpoint": resolved_endpoint} - } - return json.dumps(config, indent=2) + "\n" - - -def main(): - files_input = os.environ.get("INPUT_FILES", "").strip() - if not files_input: - print("Input 'files' cannot be empty.", file=sys.stderr) - sys.exit(1) - - config_file_name = os.environ.get("INPUT_CONFIG_FILE_NAME", "90-github-runner-log-forwarding.yaml").strip() - if "/" in config_file_name: - print("Input 'config-file-name' must not include directory separators.", file=sys.stderr) - sys.exit(1) - - config_path = os.path.join(CONFIG_DIR, config_file_name) - - if shutil.which("snap") is None: - print("Required command is missing: snap", file=sys.stderr) - sys.exit(1) - - if subprocess.run(["snap", "list", "opentelemetry-collector"], capture_output=True).returncode != 0: - print("opentelemetry-collector snap is not installed on this runner.", file=sys.stderr) - sys.exit(1) - - files = parse_files_into_list(files_input) - if not files: - print("Input 'files' must contain at least one path or glob.", file=sys.stderr) - sys.exit(1) - - resolved_endpoint = resolve_endpoint() - exporter_already_exists = check_exporter_exists() - - if not exporter_already_exists and not resolved_endpoint: - print( - f"Exporter '{EXPORTER_NAME}' was not found in scanned collector config directories and no OTLP endpoint was provided.", - file=sys.stderr, - ) - print( - "Set input 'otlp-endpoint', or expose ACTION_OTEL_EXPORTER_OTLP_ENDPOINT to this workflow.", - file=sys.stderr, - ) - sys.exit(1) - - config_content = build_config(files, resolved_endpoint, exporter_already_exists) - - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as tmp: - tmp.write(config_content) - tmp_path = tmp.name - - try: - run_as_root("mkdir", "-p", CONFIG_DIR) # create if missing, do nothing if exists - run_as_root("install", "-m", "0644", tmp_path, config_path) # owner read/write and group/other read permissions - finally: - os.unlink(tmp_path) - - print(f"Wrote log-forwarding collector config to: {config_path}") - - run_as_root("snap", "restart", "opentelemetry-collector") - print("Restarted opentelemetry-collector to apply log-forwarding config.") - - -if __name__ == "__main__": - main() diff --git a/actions/enable-log-forwarding/action.yaml b/actions/enable_log_forwarding/action.yaml similarity index 82% rename from actions/enable-log-forwarding/action.yaml rename to actions/enable_log_forwarding/action.yaml index c467104c..40f81fef 100644 --- a/actions/enable-log-forwarding/action.yaml +++ b/actions/enable_log_forwarding/action.yaml @@ -9,9 +9,9 @@ inputs: required: true otlp-endpoint: description: | - Optional gRPC endpoint for upstream OpenTelemetry Collector logs export. + Optional OTLP/gRPC endpoint for upstream OpenTelemetry Collector logs export. When not set, the action falls back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT. - Example: otel-gateway.internal:4318 + Example: otel-gateway.internal:4317 required: false default: "" config-file-name: @@ -28,4 +28,4 @@ runs: INPUT_FILES: ${{ inputs.files }} INPUT_OTLP_ENDPOINT: ${{ inputs.otlp-endpoint }} INPUT_CONFIG_FILE_NAME: ${{ inputs.config-file-name }} - run: python3 "${{ github.action_path }}/enable-log-forwarding.py" + run: python3 "${{ github.action_path }}/enable_log_forwarding.py" diff --git a/actions/enable_log_forwarding/enable_log_forwarding.py b/actions/enable_log_forwarding/enable_log_forwarding.py new file mode 100644 index 00000000..d4d09c46 --- /dev/null +++ b/actions/enable_log_forwarding/enable_log_forwarding.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +"""Configure OpenTelemetry Collector log forwarding for selected runner log files.""" + +import json +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Sequence + +CONFIG_DIR = "/etc/otelcol/config.d" +EXPORTER_NAME = "otlp_grpc" +SNAP_CMD = shutil.which("snap") +SUDO_CMD = shutil.which("sudo") + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + + +def run_as_root(*args: str) -> subprocess.CompletedProcess[bytes]: + """Run a command directly as root or through sudo when available.""" + if os.geteuid() == 0: # if running as root + return subprocess.run(args, capture_output=True, check=False) + if SUDO_CMD: # if sudo is available + return subprocess.run([SUDO_CMD, *args], capture_output=True, check=False) + logger.error("This action requires root privileges to update collector config.") + sys.exit(1) + + +def parse_files_into_list(files_input: str) -> list[str]: + """Parse comma/newline-separated file patterns into a normalized list.""" + entries = [] + # Split on commas or newlines, and strip whitespace. Ignore empty entries. + for item in re.split(r"[,\n]", files_input): + stripped = item.strip() + if stripped: + entries.append(stripped) + return entries + + +def resolve_endpoint() -> str: + """Resolve OTLP endpoint from explicit input, then workflow fallback variable.""" + # If INPUT_OTLP_ENDPOINT is not set, fall back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT + for env_var in ( + "INPUT_OTLP_ENDPOINT", + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", + ): + val = os.getenv(env_var, "").strip() + if val: + return val + return "" + + +def check_exporter_exists() -> bool: + """Return whether the configured exporter is already defined in collector config files.""" + # Check for " otlp_grpc:" exporter definition + pattern = f"^ {re.escape(EXPORTER_NAME)}:[ \t]*$" + config_dir = Path(CONFIG_DIR) + if not config_dir.is_dir(): + return False + + matcher = re.compile(pattern) + for path in config_dir.rglob("*"): + if not path.is_file(): + continue + try: + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if matcher.match(line): + return True + except OSError: + continue + return False + + +def build_config( + files: Sequence[str], resolved_endpoint: str, exporter_already_exists: bool +) -> str: + """Build a collector pipeline config fragment for opt-in log forwarding.""" + attrs = [ + ("github.repository", os.getenv("GITHUB_REPOSITORY", "unknown")), + ("github.runner.name", os.getenv("RUNNER_NAME", "unknown")), + ("github.workflow", os.getenv("GITHUB_WORKFLOW", "unknown")), + ("github.job.id", os.getenv("GITHUB_JOB", "unknown")), + ("github.run.id", os.getenv("GITHUB_RUN_ID", "unknown")), + ("github.run.attempt", os.getenv("GITHUB_RUN_ATTEMPT", "unknown")), + ] + config = { + "receivers": { + "filelog/github_runner_optin": { + "include": files, + "start_at": "end", + } + }, + "processors": { + "resource/github_runner_optin": { + "attributes": [ + {"key": key, "value": value, "action": "upsert"} + for key, value in attrs + ] + } + }, + "service": { + "pipelines": { + "logs/github_runner_optin": { + "receivers": ["filelog/github_runner_optin"], + "processors": ["resource/github_runner_optin", "batch"], + "exporters": [EXPORTER_NAME], + } + } + }, + } + if not exporter_already_exists and resolved_endpoint: + config["exporters"] = {EXPORTER_NAME: {"endpoint": resolved_endpoint}} + return json.dumps(config, indent=2) + "\n" + + +def main(): + """Validate inputs, write collector config, and restart the collector service.""" + files_input = os.getenv("INPUT_FILES", "").strip() + if not files_input: + logger.error("Input 'files' cannot be empty.") + sys.exit(1) + + config_file_name = os.getenv( + "INPUT_CONFIG_FILE_NAME", "90-github-runner-log-forwarding.yaml" + ).strip() + if ( + not config_file_name + or config_file_name in {".", ".."} + or Path(config_file_name).name != config_file_name + ): + logger.error( + "Input 'config-file-name' must be a non-empty file name without directory components.", + ) + sys.exit(1) + + config_path = str(Path(CONFIG_DIR) / config_file_name) + + if SNAP_CMD is None: + logger.error("Required command is missing: snap") + sys.exit(1) + + if ( + subprocess.run( + [SNAP_CMD, "list", "opentelemetry-collector"], + capture_output=True, + check=False, + ).returncode + != 0 + ): + logger.error("opentelemetry-collector snap is not installed on this runner.") + sys.exit(1) + + files = parse_files_into_list(files_input) + if not files: + logger.error("Input 'files' must contain at least one path or glob.") + sys.exit(1) + + resolved_endpoint = resolve_endpoint() + exporter_already_exists = check_exporter_exists() + + if not exporter_already_exists and not resolved_endpoint: + logger.error( + "Exporter '%s' was not found in scanned collector config directories " + "and no OTLP endpoint was provided.", + EXPORTER_NAME, + ) + logger.error( + "Set input 'otlp-endpoint', or expose " + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT to this workflow.", + ) + sys.exit(1) + + config_content = build_config(files, resolved_endpoint, exporter_already_exists) + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False, encoding="utf-8" + ) as tmp: + tmp.write(config_content) + tmp_path = tmp.name + + try: + mkdir_result = run_as_root( + "mkdir", "-p", CONFIG_DIR # create directory if it doesn't exist + ) + if mkdir_result.returncode != 0: + stderr = mkdir_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to create collector config directory '%s': %s", + CONFIG_DIR, + stderr or "unknown error", + ) + sys.exit(1) + + install_result = run_as_root( + "install", "-m", "0644", tmp_path, config_path # rw-r--r-- permissions + ) + if install_result.returncode != 0: + stderr = install_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to install collector config to '%s': %s", + config_path, + stderr or "unknown error", + ) + sys.exit(1) + finally: + os.unlink(tmp_path) + + logger.info("Wrote log-forwarding collector config to: %s", config_path) + + restart_result = run_as_root(SNAP_CMD, "restart", "opentelemetry-collector") + if restart_result.returncode != 0: + stderr = restart_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to restart opentelemetry-collector: %s", + stderr or "unknown error", + ) + sys.exit(1) + logger.info("Restarted opentelemetry-collector to apply log-forwarding config.") + + +if __name__ == "__main__": + main() diff --git a/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py b/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py similarity index 59% rename from actions/enable-log-forwarding/tests/test_enable_log_forwarding.py rename to actions/enable_log_forwarding/tests/test_enable_log_forwarding.py index 47ae5fb7..6f899c84 100644 --- a/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py +++ b/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py @@ -1,15 +1,18 @@ +"""Unit tests for the enable_log_forwarding action script.""" + import importlib.util import json import pathlib -import types import unittest from unittest import mock - -MODULE_PATH = pathlib.Path(__file__).resolve().parent.parent / "enable-log-forwarding.py" +MODULE_PATH = ( + pathlib.Path(__file__).resolve().parent.parent / "enable_log_forwarding.py" +) def load_module(path: pathlib.Path): + """Load the action module from file for direct function-level unit testing.""" spec = importlib.util.spec_from_file_location("enable_log_forwarding", path) if spec is None or spec.loader is None: raise RuntimeError(f"Unable to load module from {path}") @@ -22,12 +25,18 @@ def load_module(path: pathlib.Path): class TestEnableLogForwarding(unittest.TestCase): + """Test suite for parsing, endpoint resolution, exporter detection, and config generation.""" + def test_parse_files_into_list(self): + """It parses comma/newline-separated inputs and drops empty entries.""" files_input = " /var/log/a.log,\n/var/log/b.log ,, /var/log/c*.log\n" parsed = MODULE.parse_files_into_list(files_input) - self.assertEqual(parsed, ["/var/log/a.log", "/var/log/b.log", "/var/log/c*.log"]) + self.assertEqual( + parsed, ["/var/log/a.log", "/var/log/b.log", "/var/log/c*.log"] + ) def test_resolve_endpoint_prefers_input(self): + """It prefers the explicit action input endpoint over fallback env values.""" with mock.patch.dict( MODULE.os.environ, { @@ -41,6 +50,7 @@ def test_resolve_endpoint_prefers_input(self): self.assertEqual(resolved, "input-endpoint:4318") def test_resolve_endpoint_falls_back_to_action_env(self): + """It falls back to the workflow-provided endpoint when input is empty.""" with mock.patch.dict( MODULE.os.environ, { @@ -54,12 +64,18 @@ def test_resolve_endpoint_falls_back_to_action_env(self): self.assertEqual(resolved, "system-endpoint:4318") def test_check_exporter_exists_true(self): - # First call checks directory exists, second call checks grep match. - calls = [types.SimpleNamespace(returncode=0), types.SimpleNamespace(returncode=0)] - with mock.patch.object(MODULE, "run_as_root", side_effect=calls): + """It returns true when a collector config file defines the exporter name.""" + mock_file = mock.Mock(spec=pathlib.Path) + mock_file.is_file.return_value = True + mock_file.read_text.return_value = "exporters:\n otlp_grpc:\n" + + with mock.patch.object( + pathlib.Path, "is_dir", return_value=True + ), mock.patch.object(pathlib.Path, "rglob", return_value=[mock_file]): self.assertTrue(MODULE.check_exporter_exists()) def test_build_config_adds_exporter_when_missing(self): + """It adds an exporter block when no pre-existing exporter is detected.""" env = { "GITHUB_REPOSITORY": "canonical/github-runner-operators", "RUNNER_NAME": "runner-1", @@ -72,11 +88,20 @@ def test_build_config_adds_exporter_when_missing(self): raw = MODULE.build_config(["/var/log/syslog"], "otel:4318", False) config = json.loads(raw) - self.assertEqual(config["receivers"]["filelog/github_runner_optin"]["include"], ["/var/log/syslog"]) - self.assertEqual(config["service"]["pipelines"]["logs/github_runner_optin"]["exporters"], [MODULE.EXPORTER_NAME]) - self.assertEqual(config["exporters"][MODULE.EXPORTER_NAME]["endpoint"], "otel:4318") + self.assertEqual( + config["receivers"]["filelog/github_runner_optin"]["include"], + ["/var/log/syslog"], + ) + self.assertEqual( + config["service"]["pipelines"]["logs/github_runner_optin"]["exporters"], + [MODULE.EXPORTER_NAME], + ) + self.assertEqual( + config["exporters"][MODULE.EXPORTER_NAME]["endpoint"], "otel:4318" + ) def test_build_config_reuses_existing_exporter(self): + """It omits exporter creation when an exporter already exists elsewhere.""" with mock.patch.dict(MODULE.os.environ, {}, clear=False): raw = MODULE.build_config(["/var/log/syslog"], "otel:4318", True) diff --git a/docs/how-to/enable-log-forwarding.md b/docs/how-to/enable-log-forwarding.md index d5aded4f..6a9982d4 100644 --- a/docs/how-to/enable-log-forwarding.md +++ b/docs/how-to/enable-log-forwarding.md @@ -4,22 +4,30 @@ The `enable-log-forwarding` action allows workflow authors to opt in to forwardi By default, nothing is forwarded. Log forwarding starts only when this action is used in a workflow. -## Provide inputs +## Prerequisites + +- Use a self-hosted Linux runner. +- Install the `opentelemetry-collector` snap on the runner. +- Ensure the workflow can update `/etc/otelcol/config.d` with root privileges. + +## Provide inputs - `files` (required): newline or comma separated file paths or glob patterns. - `config-file-name` (optional, default `90-github-runner-log-forwarding.yaml`): generated config file name. -- `otlp-endpoint` (optional): OTLP/HTTP endpoint used to create the exporter when one is not already configured. +- `otlp-endpoint` (optional): OTLP/gRPC endpoint used to create the exporter when one is not already configured. When `otlp-endpoint` is not set, the action falls back to `ACTION_OTEL_EXPORTER_OTLP_ENDPOINT` from the workflow environment. ## Use the action +Add this snippet to a job in your workflow file (for example, `.github/workflows/ci.yaml`): + ```yaml jobs: chrony-testing: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux] steps: - - uses: canonical/github-runner-operators/actions/enable-log-forwarding@main + - uses: canonical/github-runner-operators/actions/enable_log_forwarding@main with: files: | /var/log/chrony/*.log @@ -29,24 +37,19 @@ jobs: Pin to a release tag or commit SHA in production workflows. -## Examine Loki queries +## Examine Loki queries The action adds GitHub context as resource attributes on forwarded logs: - `github.job.id` -- `github.job.name` - `github.repository` - `github.runner.name` - `github.workflow` +- `github.run.id` - `github.run.attempt` Example Loki query by workflow run id: -```logql -{github_job_id="123456789"} ``` - -## Notes - -- This action requires root privileges to write collector config. -- The `opentelemetry-collector` snap must be installed on the runner. +{github_run_id="123456789"} +``` diff --git a/tox.ini b/tox.ini index 7808cf7b..70e7628b 100644 --- a/tox.ini +++ b/tox.ini @@ -4,11 +4,12 @@ [tox] no_package = True skip_missing_interpreters = True -env_list = format, lint, static +env_list = format, lint, static, actions-lint, actions-static, actions-unit min_version = 4.0.0 [vars] tests_path = {tox_root}/charms/tests +actions_path = {tox_root}/actions [testenv] setenv = @@ -69,3 +70,28 @@ commands = --log-cli-level=INFO \ {[vars]tests_path}/integration \ {posargs} + +[testenv:actions-lint] +description = Run formatting and lint checks for Python code under actions/ +deps = + black + ruff +commands = + black --check {[vars]actions_path} + ruff check {[vars]actions_path} + +[testenv:actions-static] +description = Run static analysis for Python code under actions/ +deps = + mypy + pylint +commands = + mypy {[vars]actions_path} + pylint {[vars]actions_path} + +[testenv:actions-unit] +description = Run unit tests for Python code under actions/ +deps = + pytest +commands = + pytest -v -s {[vars]actions_path} From f15564a40e7339bb355f2bf5e58fc7ba39746266 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Fri, 24 Apr 2026 01:17:19 +0700 Subject: [PATCH 17/23] add license Co-authored-by: Copilot --- .github/workflows/enable_log_forwarding_action_tests.yaml | 5 ++--- actions/enable_log_forwarding/action.yaml | 2 ++ .../tests/test_enable_log_forwarding.py | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/enable_log_forwarding_action_tests.yaml b/.github/workflows/enable_log_forwarding_action_tests.yaml index 4761e4fe..73acd7c6 100644 --- a/.github/workflows/enable_log_forwarding_action_tests.yaml +++ b/.github/workflows/enable_log_forwarding_action_tests.yaml @@ -49,7 +49,7 @@ jobs: smoke-test-self-hosted: needs: detect-changes - if: ${{ needs.detect-changes.outputs.action == 'true' && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'self-hosted-smoke')) }} + if: ${{ needs.detect-changes.outputs.action == 'true' }} runs-on: [self-hosted-linux-amd64-noble-edge] env: TEST_CONFIG_FILE: 98-enable-log-forwarding-smoke-${{ github.run_id }}-${{ github.run_attempt }}.yaml @@ -62,7 +62,7 @@ jobs: files: | /var/log/syslog config-file-name: ${{ env.TEST_CONFIG_FILE }} - otlp-endpoint: 127.0.0.1:4318 + otlp-endpoint: 127.0.0.1:4317 - name: Verify generated config file exists run: | @@ -71,4 +71,3 @@ jobs: - name: Verify generated config contains expected receiver run: | sudo grep -q '"filelog/github_runner_optin"' /etc/otelcol/config.d/${TEST_CONFIG_FILE} - diff --git a/actions/enable_log_forwarding/action.yaml b/actions/enable_log_forwarding/action.yaml index 40f81fef..7337ed21 100644 --- a/actions/enable_log_forwarding/action.yaml +++ b/actions/enable_log_forwarding/action.yaml @@ -1,3 +1,5 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. name: Enable log forwarding description: Opt in to forward selected log files from a self-hosted GitHub runner to Loki. diff --git a/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py b/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py index 6f899c84..61404587 100644 --- a/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py +++ b/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py @@ -1,3 +1,5 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. """Unit tests for the enable_log_forwarding action script.""" import importlib.util From bc2a1e4b8832d892d1f673f5d02176d9c0a8ca7f Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Fri, 24 Apr 2026 10:09:19 +0700 Subject: [PATCH 18/23] fix docs Co-authored-by: Copilot --- docs/how-to/enable-log-forwarding.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/how-to/enable-log-forwarding.md b/docs/how-to/enable-log-forwarding.md index 6a9982d4..eeab7f6d 100644 --- a/docs/how-to/enable-log-forwarding.md +++ b/docs/how-to/enable-log-forwarding.md @@ -32,11 +32,15 @@ jobs: files: | /var/log/chrony/*.log /var/log/syslog - - run: ./run-tests.sh ``` Pin to a release tag or commit SHA in production workflows. +Use these checks to confirm forwarding: + +- Check the action step is completed and prints success messages in the workflow logs. +- Generate new log lines after the action step and query Loki to confirm they arrive. + ## Examine Loki queries The action adds GitHub context as resource attributes on forwarded logs: From 5016dcbeaf74a24df57329c4234c122fb58e56f4 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Fri, 24 Apr 2026 10:43:12 +0700 Subject: [PATCH 19/23] refactor code Co-authored-by: Copilot --- .../enable_log_forwarding/collector_config.j2 | 23 +++ .../enable_log_forwarding.py | 171 ++++++++++++------ 2 files changed, 135 insertions(+), 59 deletions(-) create mode 100644 actions/enable_log_forwarding/collector_config.j2 diff --git a/actions/enable_log_forwarding/collector_config.j2 b/actions/enable_log_forwarding/collector_config.j2 new file mode 100644 index 00000000..b48fe5ae --- /dev/null +++ b/actions/enable_log_forwarding/collector_config.j2 @@ -0,0 +1,23 @@ +{ + "receivers": { + "filelog/github_runner_optin": { + "include": {{ include_files }}, + "start_at": "end" + } + }, +{{ exporters_section }} + "processors": { + "resource/github_runner_optin": { + "attributes": {{ resource_attributes }} + } + }, + "service": { + "pipelines": { + "logs/github_runner_optin": { + "receivers": ["filelog/github_runner_optin"], + "processors": ["resource/github_runner_optin", "batch"], + "exporters": [{{ exporter_name }}] + } + } + } +} diff --git a/actions/enable_log_forwarding/enable_log_forwarding.py b/actions/enable_log_forwarding/enable_log_forwarding.py index d4d09c46..7e7aed3f 100644 --- a/actions/enable_log_forwarding/enable_log_forwarding.py +++ b/actions/enable_log_forwarding/enable_log_forwarding.py @@ -11,11 +11,13 @@ import subprocess import sys import tempfile +import textwrap from pathlib import Path from typing import Sequence CONFIG_DIR = "/etc/otelcol/config.d" EXPORTER_NAME = "otlp_grpc" +CONFIG_TEMPLATE_PATH = Path(__file__).with_name("collector_config.j2") SNAP_CMD = shutil.which("snap") SUDO_CMD = shutil.which("sudo") @@ -78,10 +80,22 @@ def check_exporter_exists() -> bool: return False -def build_config( - files: Sequence[str], resolved_endpoint: str, exporter_already_exists: bool -) -> str: - """Build a collector pipeline config fragment for opt-in log forwarding.""" +def render_template(template_path: Path, context: dict[str, str]) -> str: + """Render a minimal Jinja-style template with {{ var }} placeholders.""" + template = template_path.read_text(encoding="utf-8") + pattern = re.compile(r"{{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*}}") + + def replacer(match: re.Match[str]) -> str: + key = match.group(1) + if key not in context: + raise KeyError(f"Missing template variable: {key}") + return context[key] + + return pattern.sub(replacer, template) + + +def build_resource_attributes() -> list[dict[str, str]]: + """Build static GitHub resource attributes attached to forwarded logs.""" attrs = [ ("github.repository", os.getenv("GITHUB_REPOSITORY", "unknown")), ("github.runner.name", os.getenv("RUNNER_NAME", "unknown")), @@ -90,43 +104,54 @@ def build_config( ("github.run.id", os.getenv("GITHUB_RUN_ID", "unknown")), ("github.run.attempt", os.getenv("GITHUB_RUN_ATTEMPT", "unknown")), ] - config = { - "receivers": { - "filelog/github_runner_optin": { - "include": files, - "start_at": "end", - } - }, - "processors": { - "resource/github_runner_optin": { - "attributes": [ - {"key": key, "value": value, "action": "upsert"} - for key, value in attrs - ] - } - }, - "service": { - "pipelines": { - "logs/github_runner_optin": { - "receivers": ["filelog/github_runner_optin"], - "processors": ["resource/github_runner_optin", "batch"], - "exporters": [EXPORTER_NAME], - } + return [{"key": key, "value": value, "action": "upsert"} for key, value in attrs] + + +def build_exporters_section( + resolved_endpoint: str, exporter_already_exists: bool +) -> str: + """Build the optional exporters JSON fragment for the template.""" + if exporter_already_exists: + return "" + + exporters_block = { + "exporters": { + EXPORTER_NAME: { + "endpoint": resolved_endpoint, } - }, + } } - if not exporter_already_exists and resolved_endpoint: - config["exporters"] = {EXPORTER_NAME: {"endpoint": resolved_endpoint}} - return json.dumps(config, indent=2) + "\n" + block = json.dumps(exporters_block, indent=2) + inner = block.strip()[1:-1].strip() + return textwrap.indent(inner, " ") + ",\n" -def main(): - """Validate inputs, write collector config, and restart the collector service.""" +def build_config( + files: Sequence[str], resolved_endpoint: str, exporter_already_exists: bool +) -> str: + """Build a collector pipeline config fragment for opt-in log forwarding.""" + context = { + "include_files": json.dumps(list(files)), + "resource_attributes": json.dumps(build_resource_attributes()), + "exporters_section": build_exporters_section( + resolved_endpoint, exporter_already_exists + ), + "exporter_name": json.dumps(EXPORTER_NAME), + } + return render_template(CONFIG_TEMPLATE_PATH, context) + + +def read_files_input() -> str: + """Read and validate the required files input.""" files_input = os.getenv("INPUT_FILES", "").strip() if not files_input: logger.error("Input 'files' cannot be empty.") sys.exit(1) + return files_input + +def resolve_config_path() -> str: + """Resolve and validate the destination config file path.""" config_file_name = os.getenv( "INPUT_CONFIG_FILE_NAME", "90-github-runner-log-forwarding.yaml" ).strip() @@ -140,45 +165,46 @@ def main(): ) sys.exit(1) - config_path = str(Path(CONFIG_DIR) / config_file_name) + return str(Path(CONFIG_DIR) / config_file_name) + +def ensure_collector_is_available() -> None: + """Check snap prerequisites needed to configure the collector.""" if SNAP_CMD is None: logger.error("Required command is missing: snap") sys.exit(1) - if ( - subprocess.run( - [SNAP_CMD, "list", "opentelemetry-collector"], - capture_output=True, - check=False, - ).returncode - != 0 - ): + snap_list_result = subprocess.run( + [SNAP_CMD, "list", "opentelemetry-collector"], + capture_output=True, + check=False, + ) + if snap_list_result.returncode != 0: logger.error("opentelemetry-collector snap is not installed on this runner.") sys.exit(1) - files = parse_files_into_list(files_input) - if not files: - logger.error("Input 'files' must contain at least one path or glob.") - sys.exit(1) - resolved_endpoint = resolve_endpoint() - exporter_already_exists = check_exporter_exists() +def validate_exporter_configuration( + resolved_endpoint: str, exporter_already_exists: bool +) -> None: + """Ensure there is enough exporter information to build a working config.""" + if exporter_already_exists or resolved_endpoint: + return - if not exporter_already_exists and not resolved_endpoint: - logger.error( - "Exporter '%s' was not found in scanned collector config directories " - "and no OTLP endpoint was provided.", - EXPORTER_NAME, - ) - logger.error( - "Set input 'otlp-endpoint', or expose " - "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT to this workflow.", - ) - sys.exit(1) + logger.error( + "Exporter '%s' was not found in scanned collector config directories " + "and no OTLP endpoint was provided.", + EXPORTER_NAME, + ) + logger.error( + "Set input 'otlp-endpoint', or expose " + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT to this workflow.", + ) + sys.exit(1) - config_content = build_config(files, resolved_endpoint, exporter_already_exists) +def write_collector_config(config_content: str, config_path: str) -> None: + """Write generated config to /etc/otelcol/config.d via root privileges.""" with tempfile.NamedTemporaryFile( mode="w", suffix=".yaml", delete=False, encoding="utf-8" ) as tmp: @@ -214,6 +240,13 @@ def main(): logger.info("Wrote log-forwarding collector config to: %s", config_path) + +def restart_collector() -> None: + """Restart collector service so new config is loaded.""" + if SNAP_CMD is None: + logger.error("Required command is missing: snap") + sys.exit(1) + restart_result = run_as_root(SNAP_CMD, "restart", "opentelemetry-collector") if restart_result.returncode != 0: stderr = restart_result.stderr.decode(errors="replace").strip() @@ -225,5 +258,25 @@ def main(): logger.info("Restarted opentelemetry-collector to apply log-forwarding config.") +def main(): + """Validate inputs, write collector config, and restart the collector service.""" + files_input = read_files_input() + config_path = resolve_config_path() + ensure_collector_is_available() + + files = parse_files_into_list(files_input) + if not files: + logger.error("Input 'files' must contain at least one path or glob.") + sys.exit(1) + + resolved_endpoint = resolve_endpoint() + exporter_already_exists = check_exporter_exists() + validate_exporter_configuration(resolved_endpoint, exporter_already_exists) + + config_content = build_config(files, resolved_endpoint, exporter_already_exists) + write_collector_config(config_content, config_path) + restart_collector() + + if __name__ == "__main__": main() From a920d9a5e8030d7107af992f7c72cd7c058a6e2f Mon Sep 17 00:00:00 2001 From: florentianayuwono <76247368+florentianayuwono@users.noreply.github.com> Date: Mon, 27 Apr 2026 13:26:30 +0700 Subject: [PATCH 20/23] Apply suggestions from code review Co-authored-by: Erin Conley --- docs/how-to/enable-log-forwarding.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/how-to/enable-log-forwarding.md b/docs/how-to/enable-log-forwarding.md index eeab7f6d..d7c3faf7 100644 --- a/docs/how-to/enable-log-forwarding.md +++ b/docs/how-to/enable-log-forwarding.md @@ -12,6 +12,8 @@ By default, nothing is forwarded. Log forwarding starts only when this action is ## Provide inputs +To enable log forwarding, set the following inputs in your workflow file as required by your setup: + - `files` (required): newline or comma separated file paths or glob patterns. - `config-file-name` (optional, default `90-github-runner-log-forwarding.yaml`): generated config file name. - `otlp-endpoint` (optional): OTLP/gRPC endpoint used to create the exporter when one is not already configured. @@ -34,7 +36,7 @@ jobs: /var/log/syslog ``` -Pin to a release tag or commit SHA in production workflows. +Pin the action to a release tag or commit SHA in production workflows. Use these checks to confirm forwarding: From dc8fd5a09c585f347974de5884a5e7b409fcfda0 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Wed, 29 Apr 2026 11:20:46 +0700 Subject: [PATCH 21/23] address code reviews Co-authored-by: Copilot --- .../enable_log_forwarding_action_tests.yaml | 23 +- .../action.yaml | 17 +- .../enable_log_forwarding.py | 315 ++++++++++++++++++ .../tests/test_enable_log_forwarding.py | 237 +++++++++++++ .../enable_log_forwarding/collector_config.j2 | 23 -- .../enable_log_forwarding.py | 282 ---------------- .../tests/test_enable_log_forwarding.py | 115 ------- docs/conf.py | 6 + docs/how-to/enable-log-forwarding.md | 13 +- pyproject.toml | 13 + tox.ini | 5 + 11 files changed, 618 insertions(+), 431 deletions(-) rename actions/{enable_log_forwarding => enable-log-forwarding}/action.yaml (61%) create mode 100644 actions/enable-log-forwarding/enable_log_forwarding.py create mode 100644 actions/enable-log-forwarding/tests/test_enable_log_forwarding.py delete mode 100644 actions/enable_log_forwarding/collector_config.j2 delete mode 100644 actions/enable_log_forwarding/enable_log_forwarding.py delete mode 100644 actions/enable_log_forwarding/tests/test_enable_log_forwarding.py create mode 100644 pyproject.toml diff --git a/.github/workflows/enable_log_forwarding_action_tests.yaml b/.github/workflows/enable_log_forwarding_action_tests.yaml index 73acd7c6..d91f3e37 100644 --- a/.github/workflows/enable_log_forwarding_action_tests.yaml +++ b/.github/workflows/enable_log_forwarding_action_tests.yaml @@ -23,7 +23,7 @@ jobs: with: filters: | action: - - 'actions/enable_log_forwarding/**' + - 'actions/enable-log-forwarding/**' - '.github/workflows/enable_log_forwarding_action_tests.yaml' test-action: @@ -52,12 +52,12 @@ jobs: if: ${{ needs.detect-changes.outputs.action == 'true' }} runs-on: [self-hosted-linux-amd64-noble-edge] env: - TEST_CONFIG_FILE: 98-enable-log-forwarding-smoke-${{ github.run_id }}-${{ github.run_attempt }}.yaml + TEST_CONFIG_FILE: 91-enable-log-forwarding-smoke-${{ github.run_id }}-${{ github.run_attempt }}.yaml steps: - uses: actions/checkout@v6 - name: Run enable log forwarding action - uses: ./actions/enable_log_forwarding + uses: ./actions/enable-log-forwarding with: files: | /var/log/syslog @@ -71,3 +71,20 @@ jobs: - name: Verify generated config contains expected receiver run: | sudo grep -q '"filelog/github_runner_optin"' /etc/otelcol/config.d/${TEST_CONFIG_FILE} + + smoke-test-github-hosted: + needs: detect-changes + if: ${{ needs.detect-changes.outputs.action == 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Run enable log forwarding action on github-hosted runner + uses: ./actions/enable-log-forwarding + with: + files: | + /var/log/syslog + + - name: Verify workflow continues after github-hosted no-op + run: | + echo "enable-log-forwarding exited successfully on github-hosted runner" diff --git a/actions/enable_log_forwarding/action.yaml b/actions/enable-log-forwarding/action.yaml similarity index 61% rename from actions/enable_log_forwarding/action.yaml rename to actions/enable-log-forwarding/action.yaml index 7337ed21..007c4ebe 100644 --- a/actions/enable_log_forwarding/action.yaml +++ b/actions/enable-log-forwarding/action.yaml @@ -6,24 +6,33 @@ description: Opt in to forward selected log files from a self-hosted GitHub runn inputs: files: description: | - Newline or comma separated list of file paths or glob patterns to forward. - Example: /var/log/chrony/*.log + Newline or comma-separated list of file paths and glob patterns to forward. + Examples: + newline-separated: /var/log/syslog\n/var/log/kern.log + comma-separated: /var/log/syslog,/var/log/kern.log + glob: /var/log/chrony/*.log required: true otlp-endpoint: description: | Optional OTLP/gRPC endpoint for upstream OpenTelemetry Collector logs export. - When not set, the action falls back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT. + When not set, the action falls back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT, + which is injected by Canonical self-hosted runners. Example: otel-gateway.internal:4317 required: false default: "" config-file-name: - description: File name for the generated collector fragment. + description: | + File name for the generated collector fragment under /etc/otelcol/config.d. + Use this to control merge/load priority relative to other fragments. required: false default: 90-github-runner-log-forwarding.yaml runs: using: composite steps: + - name: Install action Python dependencies + shell: bash + run: pip install pyyaml --quiet - name: Configure collector for opt-in file log forwarding shell: bash env: diff --git a/actions/enable-log-forwarding/enable_log_forwarding.py b/actions/enable-log-forwarding/enable_log_forwarding.py new file mode 100644 index 00000000..c3d49105 --- /dev/null +++ b/actions/enable-log-forwarding/enable_log_forwarding.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +"""Configure OpenTelemetry Collector log forwarding for selected runner log files.""" + +import json +import logging +import os +import re +import shutil +import subprocess # nosec B404 +import sys +import tempfile +from pathlib import Path +from typing import Sequence + +import yaml + +CONFIG_DIR = "/etc/otelcol/config.d" +EXPORTER_NAME = "otlp_grpc" +SNAP_CMD = Path("/usr/bin/snap") +SUDO_CMD = Path("/usr/bin/sudo") +MKDIR_CMD = Path("/usr/bin/mkdir") +CP_CMD = Path("/usr/bin/cp") +CHMOD_CMD = Path("/usr/bin/chmod") +FILES_SPLIT_PATTERN = re.compile(r"[,\n]") # character class: split on comma or newline + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + + +def run_as_root(*args: str) -> subprocess.CompletedProcess[bytes]: + """Run a command directly as root or through sudo when available.""" + if os.geteuid() == 0: # if running as root + return subprocess.run(args, capture_output=True, check=False) # nosec B603 + if SUDO_CMD.is_file(): # if sudo is available + return subprocess.run( + [str(SUDO_CMD), *args], capture_output=True, check=False + ) # nosec B603 + logger.error("This action requires root privileges to update collector config.") + sys.exit(1) + + +def parse_files_into_list(files_input: str) -> list[str]: + """Parse comma/newline-separated file patterns into a normalized non-empty list.""" + entries = [] + for item in FILES_SPLIT_PATTERN.split(files_input): + stripped = item.strip() + if stripped: + entries.append(stripped) + + if not entries: + logger.error("Input 'files' must contain at least one path or glob.") + sys.exit(1) + + return entries + + +def resolve_endpoint() -> str: + """Resolve OTLP endpoint from explicit input, + falling back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT.""" + for env_var in ( + "INPUT_OTLP_ENDPOINT", + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", + ): + val = os.getenv(env_var, "").strip() + if val: + return val + + logger.error( + "No OTLP endpoint was provided. Set input 'otlp-endpoint', or expose " + "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT to this workflow.", + ) + sys.exit(1) + + +def is_github_hosted_runner() -> bool: + """Return whether this action is running on a GitHub-hosted runner.""" + return os.getenv("RUNNER_ENVIRONMENT", "").strip().lower() == "github-hosted" + + +def exporter_exists_in_config_dir( + exporter_name: str, config_dir: str, exclude_path: str +) -> bool: + """Check if an exporter with the given name is already defined in another config fragment.""" + config_dir_path = Path(config_dir) + if not config_dir_path.is_dir(): + return False + for config_file in config_dir_path.iterdir(): + if not config_file.is_file() or config_file.suffix not in { + ".yaml", + ".yml", + ".json", + }: + continue + if str(config_file) == exclude_path: + continue + try: + content = yaml.safe_load(config_file.read_text(encoding="utf-8")) + except (yaml.YAMLError, OSError): + continue + if isinstance(content, dict): + exporters = content.get("exporters", {}) + if isinstance(exporters, dict) and exporter_name in exporters: + return True + return False + + +def build_resource_attributes() -> list[dict[str, str]]: + """Build static GitHub resource attributes attached to forwarded logs.""" + attrs = [ + ("github.repository", os.getenv("GITHUB_REPOSITORY", "unknown")), + ("github.runner", os.getenv("RUNNER_NAME", "unknown")), + ("github.workflow", os.getenv("GITHUB_WORKFLOW", "unknown")), + ("github.job", os.getenv("GITHUB_JOB", "unknown")), + ("github.run.id", os.getenv("GITHUB_RUN_ID", "unknown")), + ("github.run.attempt", os.getenv("GITHUB_RUN_ATTEMPT", "unknown")), + ] + return [{"key": key, "value": value, "action": "upsert"} for key, value in attrs] + + +def build_config( + files: Sequence[str], + resolved_endpoint: str, + exporter_name: str, + define_exporter: bool = True, +) -> str: + """Build a collector pipeline config fragment for opt-in log forwarding.""" + config: dict = { + "receivers": { + "filelog/github_runner_optin": { + "include": list(files), + "start_at": "end", + } + }, + "processors": { + "resource/github_runner_optin": { + "attributes": build_resource_attributes(), + } + }, + "service": { + "pipelines": { + "logs/github_runner_optin": { + "receivers": ["filelog/github_runner_optin"], + "processors": ["resource/github_runner_optin", "batch"], + "exporters": [exporter_name], + } + } + }, + } + if define_exporter: + config["exporters"] = { + exporter_name: { + "endpoint": resolved_endpoint, + } + } + + return json.dumps(config, indent=2) + + +def read_files_input() -> str: + """Read and validate the required files input.""" + files_input = os.getenv("INPUT_FILES", "").strip() + if not files_input: + logger.error("Input 'files' cannot be empty.") + sys.exit(1) + return files_input + + +def read_config_file_name() -> str: + """Read and validate the destination config file name.""" + config_file_name = os.getenv( + "INPUT_CONFIG_FILE_NAME", "90-github-runner-log-forwarding.yaml" + ).strip() + if ( + not config_file_name + or config_file_name in {".", ".."} + or Path(config_file_name).name != config_file_name + ): + logger.error( + "Input 'config-file-name' must be a non-empty file name without directory components.", + ) + sys.exit(1) + + return config_file_name + + +def ensure_collector_is_available() -> None: + """Ensure the opentelemetry-collector snap is available on the runner.""" + if not SNAP_CMD.is_file(): + logger.error("Required command is missing: snap") + sys.exit(1) + + snap_list_result = subprocess.run( # nosec B603 + [str(SNAP_CMD), "list", "opentelemetry-collector"], + capture_output=True, + check=False, + ) + if snap_list_result.returncode == 0: + return + + logger.info("opentelemetry-collector is not installed; attempting installation.") + install_result = run_as_root(str(SNAP_CMD), "install", "opentelemetry-collector") + if install_result.returncode != 0: + stderr = install_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to install opentelemetry-collector snap: %s", + stderr or "unknown error", + ) + sys.exit(1) + logger.info("Installed opentelemetry-collector snap.") + + +def write_collector_config(config_content: str, config_path: str) -> None: + """Write generated config to /etc/otelcol/config.d via root privileges.""" + config_path_obj = Path(config_path) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False, encoding="utf-8" + ) as tmp: + tmp.write(config_content) + tmp_path = Path(tmp.name) + + try: + if os.geteuid() == 0: + config_path_obj.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(tmp_path, config_path_obj) + config_path_obj.chmod(0o644) + else: + mkdir_result = run_as_root(str(MKDIR_CMD), "-p", CONFIG_DIR) + if mkdir_result.returncode != 0: + stderr = mkdir_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to create collector config directory '%s': %s", + CONFIG_DIR, + stderr or "unknown error", + ) + sys.exit(1) + + copy_result = run_as_root(str(CP_CMD), str(tmp_path), str(config_path_obj)) + if copy_result.returncode != 0: + stderr = copy_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to copy collector config to '%s': %s", + config_path, + stderr or "unknown error", + ) + sys.exit(1) + + chmod_result = run_as_root(str(CHMOD_CMD), "0644", str(config_path_obj)) + if chmod_result.returncode != 0: + stderr = chmod_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to set collector config permissions on '%s': %s", + config_path, + stderr or "unknown error", + ) + sys.exit(1) + finally: + tmp_path.unlink(missing_ok=True) + + logger.info("Wrote log-forwarding collector config to: %s", config_path) + + +def log_generated_config(config_content: str) -> None: + """Emit generated config content in grouped GitHub Actions logs.""" + print("::group::Generated collector config") + print(config_content) + print("::endgroup::") + + +def restart_collector() -> None: + """Restart collector service so new config is loaded.""" + if not SNAP_CMD.is_file(): + logger.error("Required command is missing: snap") + sys.exit(1) + + restart_result = run_as_root(str(SNAP_CMD), "restart", "opentelemetry-collector") + if restart_result.returncode != 0: + stderr = restart_result.stderr.decode(errors="replace").strip() + logger.error( + "Failed to restart opentelemetry-collector: %s", + stderr or "unknown error", + ) + sys.exit(1) + logger.info("Restarted opentelemetry-collector to apply log-forwarding config.") + + +def main(): + """Validate inputs, write collector config, and restart the collector service.""" + if is_github_hosted_runner(): + logger.info("GitHub-hosted runner detected; skipping collector configuration.") + return + + files_input = read_files_input() + config_file_name = read_config_file_name() + config_path = str(Path(CONFIG_DIR) / config_file_name) + ensure_collector_is_available() + + files = parse_files_into_list(files_input) + + resolved_endpoint = resolve_endpoint() + define_exporter = not exporter_exists_in_config_dir( + EXPORTER_NAME, CONFIG_DIR, config_path + ) + + config_content = build_config( + files, resolved_endpoint, EXPORTER_NAME, define_exporter + ) + log_generated_config(config_content) + write_collector_config(config_content, config_path) + restart_collector() + + +if __name__ == "__main__": + main() diff --git a/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py b/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py new file mode 100644 index 00000000..e86a1bd8 --- /dev/null +++ b/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py @@ -0,0 +1,237 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. +"""Unit tests for the enable_log_forwarding action script.""" + +import importlib +import json +import pathlib +import sys +import tempfile +from typing import Any + +ACTION_DIR = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ACTION_DIR)) + +module: Any = importlib.import_module("enable_log_forwarding") + + +def test_parse_files_into_list(): + """ + arrange: prepare comma/newline-separated file input with extra empty tokens. + act: parse the files input into a normalized list. + assert: parsed output keeps only non-empty, trimmed file paths. + """ + # Arrange + files_input = " /var/log/a.log,\n/var/log/b.log ,, /var/log/c*.log\n" + + # Act + parsed = module.parse_files_into_list(files_input) + + # Assert + assert parsed == ["/var/log/a.log", "/var/log/b.log", "/var/log/c*.log"] + + +def test_resolve_endpoint_prefers_input(monkeypatch): + """ + arrange: set both explicit input and fallback endpoint environment variables. + act: resolve the endpoint used by the action. + assert: explicit input endpoint takes precedence over fallback. + """ + # Arrange + monkeypatch.setenv("INPUT_OTLP_ENDPOINT", "input-endpoint:4318") + monkeypatch.setenv("ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", "system-endpoint:4318") + + # Act + resolved = module.resolve_endpoint() + + # Assert + assert resolved == "input-endpoint:4318" + + +def test_resolve_endpoint_falls_back_to_action_env(monkeypatch): + """ + arrange: set empty explicit input and a workflow-provided fallback endpoint. + act: resolve the endpoint used by the action. + assert: fallback workflow endpoint is returned. + """ + # Arrange + monkeypatch.setenv("INPUT_OTLP_ENDPOINT", "") + monkeypatch.setenv("ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", "system-endpoint:4318") + + # Act + resolved = module.resolve_endpoint() + + # Assert + assert resolved == "system-endpoint:4318" + + +def test_is_github_hosted_runner_detects_github_hosted(monkeypatch): + """ + arrange: set RUNNER_ENVIRONMENT to github-hosted. + act: check whether runner is github-hosted. + assert: returns True. + """ + # Arrange + monkeypatch.setenv("RUNNER_ENVIRONMENT", "github-hosted") + + # Act + is_github_hosted = module.is_github_hosted_runner() + + # Assert + assert is_github_hosted is True + + +def test_is_github_hosted_runner_detects_non_github_hosted(monkeypatch): + """ + arrange: set RUNNER_ENVIRONMENT to self-hosted. + act: check whether runner is github-hosted. + assert: returns False. + """ + # Arrange + monkeypatch.setenv("RUNNER_ENVIRONMENT", "self-hosted") + + # Act + is_github_hosted = module.is_github_hosted_runner() + + # Assert + assert is_github_hosted is False + + +def test_main_skips_configuration_on_github_hosted(monkeypatch): + """ + arrange: set github-hosted environment and guard setup functions. + act: run main. + assert: setup functions are not called and action exits successfully. + """ + # Arrange + monkeypatch.setenv("RUNNER_ENVIRONMENT", "github-hosted") + + def _should_not_be_called() -> str: + raise AssertionError("read_files_input should not be called") + + monkeypatch.setattr(module, "read_files_input", _should_not_be_called) + + # Act + module.main() + + +def test_build_config_adds_exporter_when_missing(monkeypatch): + """ + arrange: define GitHub metadata env vars. + act: build and parse collector config for selected log files. + assert: config includes receiver paths, pipeline exporter, and exporter endpoint. + """ + # Arrange + monkeypatch.setenv("GITHUB_REPOSITORY", "canonical/github-runner-operators") + monkeypatch.setenv("RUNNER_NAME", "runner-1") + monkeypatch.setenv("GITHUB_WORKFLOW", "CI") + monkeypatch.setenv("GITHUB_JOB", "test") + monkeypatch.setenv("GITHUB_RUN_ID", "123") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + + # Act + raw = module.build_config(["/var/log/syslog"], "otel:4318", module.EXPORTER_NAME) + config = json.loads(raw) + + # Assert + assert config["receivers"]["filelog/github_runner_optin"]["include"] == [ + "/var/log/syslog" + ] + assert config["service"]["pipelines"]["logs/github_runner_optin"]["exporters"] == [ + module.EXPORTER_NAME + ] + assert config["exporters"][module.EXPORTER_NAME]["endpoint"] == "otel:4318" + + +def test_build_config_skips_exporter_when_define_exporter_is_false(monkeypatch): + """ + arrange: define GitHub metadata env vars. + act: build config with define_exporter=False. + assert: config has no exporters block but still references the exporter in the pipeline. + """ + # Arrange + monkeypatch.setenv("GITHUB_REPOSITORY", "canonical/github-runner-operators") + monkeypatch.setenv("RUNNER_NAME", "runner-1") + monkeypatch.setenv("GITHUB_WORKFLOW", "CI") + monkeypatch.setenv("GITHUB_JOB", "test") + monkeypatch.setenv("GITHUB_RUN_ID", "123") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + + # Act + raw = module.build_config( + ["/var/log/syslog"], "otel:4318", module.EXPORTER_NAME, define_exporter=False + ) + config = json.loads(raw) + + # Assert + assert "exporters" not in config + assert config["service"]["pipelines"]["logs/github_runner_optin"]["exporters"] == [ + module.EXPORTER_NAME + ] + + +def test_exporter_exists_in_config_dir_finds_exporter(): + """ + arrange: write a YAML config file containing the fixed exporter name. + act: check whether that exporter exists in the config directory. + assert: returns True when the exporter is present in another file. + """ + # Arrange + with tempfile.TemporaryDirectory() as config_dir: + existing = pathlib.Path(config_dir) / "91-other.yaml" + existing.write_text( + f"exporters:\n {module.EXPORTER_NAME}:\n endpoint: otel:4317\n" + ) + exclude = str(pathlib.Path(config_dir) / "91-optin.logs.yaml") + + # Act + found = module.exporter_exists_in_config_dir( + module.EXPORTER_NAME, config_dir, exclude + ) + + # Assert + assert found is True + + +def test_exporter_exists_in_config_dir_ignores_exclude_path(): + """ + arrange: write a YAML config file with the exporter, but mark it as the excluded path. + act: check whether the exporter exists excluding that file. + assert: returns False since the only matching file is excluded. + """ + # Arrange + with tempfile.TemporaryDirectory() as config_dir: + target = pathlib.Path(config_dir) / "90-github-runner-log-forwarding.yaml" + target.write_text( + f"exporters:\n {module.EXPORTER_NAME}:\n endpoint: otel:4317\n" + ) + + # Act + found = module.exporter_exists_in_config_dir( + module.EXPORTER_NAME, config_dir, str(target) + ) + + # Assert + assert found is False + + +def test_resolve_endpoint_exits_when_no_endpoint_set(monkeypatch): + """ + arrange: ensure both endpoint environment variables are unset. + act: resolve the endpoint. + assert: exits with status code 1 when no endpoint is available. + """ + # Arrange + monkeypatch.delenv("INPUT_OTLP_ENDPOINT", raising=False) + monkeypatch.delenv("ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", raising=False) + + # Act + try: + module.resolve_endpoint() + except SystemExit as error: + exit_code = error.code + else: + exit_code = None + + # Assert + assert exit_code == 1 diff --git a/actions/enable_log_forwarding/collector_config.j2 b/actions/enable_log_forwarding/collector_config.j2 deleted file mode 100644 index b48fe5ae..00000000 --- a/actions/enable_log_forwarding/collector_config.j2 +++ /dev/null @@ -1,23 +0,0 @@ -{ - "receivers": { - "filelog/github_runner_optin": { - "include": {{ include_files }}, - "start_at": "end" - } - }, -{{ exporters_section }} - "processors": { - "resource/github_runner_optin": { - "attributes": {{ resource_attributes }} - } - }, - "service": { - "pipelines": { - "logs/github_runner_optin": { - "receivers": ["filelog/github_runner_optin"], - "processors": ["resource/github_runner_optin", "batch"], - "exporters": [{{ exporter_name }}] - } - } - } -} diff --git a/actions/enable_log_forwarding/enable_log_forwarding.py b/actions/enable_log_forwarding/enable_log_forwarding.py deleted file mode 100644 index 7e7aed3f..00000000 --- a/actions/enable_log_forwarding/enable_log_forwarding.py +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. -"""Configure OpenTelemetry Collector log forwarding for selected runner log files.""" - -import json -import logging -import os -import re -import shutil -import subprocess -import sys -import tempfile -import textwrap -from pathlib import Path -from typing import Sequence - -CONFIG_DIR = "/etc/otelcol/config.d" -EXPORTER_NAME = "otlp_grpc" -CONFIG_TEMPLATE_PATH = Path(__file__).with_name("collector_config.j2") -SNAP_CMD = shutil.which("snap") -SUDO_CMD = shutil.which("sudo") - -logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") -logger = logging.getLogger(__name__) - - -def run_as_root(*args: str) -> subprocess.CompletedProcess[bytes]: - """Run a command directly as root or through sudo when available.""" - if os.geteuid() == 0: # if running as root - return subprocess.run(args, capture_output=True, check=False) - if SUDO_CMD: # if sudo is available - return subprocess.run([SUDO_CMD, *args], capture_output=True, check=False) - logger.error("This action requires root privileges to update collector config.") - sys.exit(1) - - -def parse_files_into_list(files_input: str) -> list[str]: - """Parse comma/newline-separated file patterns into a normalized list.""" - entries = [] - # Split on commas or newlines, and strip whitespace. Ignore empty entries. - for item in re.split(r"[,\n]", files_input): - stripped = item.strip() - if stripped: - entries.append(stripped) - return entries - - -def resolve_endpoint() -> str: - """Resolve OTLP endpoint from explicit input, then workflow fallback variable.""" - # If INPUT_OTLP_ENDPOINT is not set, fall back to ACTION_OTEL_EXPORTER_OTLP_ENDPOINT - for env_var in ( - "INPUT_OTLP_ENDPOINT", - "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", - ): - val = os.getenv(env_var, "").strip() - if val: - return val - return "" - - -def check_exporter_exists() -> bool: - """Return whether the configured exporter is already defined in collector config files.""" - # Check for " otlp_grpc:" exporter definition - pattern = f"^ {re.escape(EXPORTER_NAME)}:[ \t]*$" - config_dir = Path(CONFIG_DIR) - if not config_dir.is_dir(): - return False - - matcher = re.compile(pattern) - for path in config_dir.rglob("*"): - if not path.is_file(): - continue - try: - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): - if matcher.match(line): - return True - except OSError: - continue - return False - - -def render_template(template_path: Path, context: dict[str, str]) -> str: - """Render a minimal Jinja-style template with {{ var }} placeholders.""" - template = template_path.read_text(encoding="utf-8") - pattern = re.compile(r"{{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*}}") - - def replacer(match: re.Match[str]) -> str: - key = match.group(1) - if key not in context: - raise KeyError(f"Missing template variable: {key}") - return context[key] - - return pattern.sub(replacer, template) - - -def build_resource_attributes() -> list[dict[str, str]]: - """Build static GitHub resource attributes attached to forwarded logs.""" - attrs = [ - ("github.repository", os.getenv("GITHUB_REPOSITORY", "unknown")), - ("github.runner.name", os.getenv("RUNNER_NAME", "unknown")), - ("github.workflow", os.getenv("GITHUB_WORKFLOW", "unknown")), - ("github.job.id", os.getenv("GITHUB_JOB", "unknown")), - ("github.run.id", os.getenv("GITHUB_RUN_ID", "unknown")), - ("github.run.attempt", os.getenv("GITHUB_RUN_ATTEMPT", "unknown")), - ] - return [{"key": key, "value": value, "action": "upsert"} for key, value in attrs] - - -def build_exporters_section( - resolved_endpoint: str, exporter_already_exists: bool -) -> str: - """Build the optional exporters JSON fragment for the template.""" - if exporter_already_exists: - return "" - - exporters_block = { - "exporters": { - EXPORTER_NAME: { - "endpoint": resolved_endpoint, - } - } - } - block = json.dumps(exporters_block, indent=2) - inner = block.strip()[1:-1].strip() - return textwrap.indent(inner, " ") + ",\n" - - -def build_config( - files: Sequence[str], resolved_endpoint: str, exporter_already_exists: bool -) -> str: - """Build a collector pipeline config fragment for opt-in log forwarding.""" - context = { - "include_files": json.dumps(list(files)), - "resource_attributes": json.dumps(build_resource_attributes()), - "exporters_section": build_exporters_section( - resolved_endpoint, exporter_already_exists - ), - "exporter_name": json.dumps(EXPORTER_NAME), - } - return render_template(CONFIG_TEMPLATE_PATH, context) - - -def read_files_input() -> str: - """Read and validate the required files input.""" - files_input = os.getenv("INPUT_FILES", "").strip() - if not files_input: - logger.error("Input 'files' cannot be empty.") - sys.exit(1) - return files_input - - -def resolve_config_path() -> str: - """Resolve and validate the destination config file path.""" - config_file_name = os.getenv( - "INPUT_CONFIG_FILE_NAME", "90-github-runner-log-forwarding.yaml" - ).strip() - if ( - not config_file_name - or config_file_name in {".", ".."} - or Path(config_file_name).name != config_file_name - ): - logger.error( - "Input 'config-file-name' must be a non-empty file name without directory components.", - ) - sys.exit(1) - - return str(Path(CONFIG_DIR) / config_file_name) - - -def ensure_collector_is_available() -> None: - """Check snap prerequisites needed to configure the collector.""" - if SNAP_CMD is None: - logger.error("Required command is missing: snap") - sys.exit(1) - - snap_list_result = subprocess.run( - [SNAP_CMD, "list", "opentelemetry-collector"], - capture_output=True, - check=False, - ) - if snap_list_result.returncode != 0: - logger.error("opentelemetry-collector snap is not installed on this runner.") - sys.exit(1) - - -def validate_exporter_configuration( - resolved_endpoint: str, exporter_already_exists: bool -) -> None: - """Ensure there is enough exporter information to build a working config.""" - if exporter_already_exists or resolved_endpoint: - return - - logger.error( - "Exporter '%s' was not found in scanned collector config directories " - "and no OTLP endpoint was provided.", - EXPORTER_NAME, - ) - logger.error( - "Set input 'otlp-endpoint', or expose " - "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT to this workflow.", - ) - sys.exit(1) - - -def write_collector_config(config_content: str, config_path: str) -> None: - """Write generated config to /etc/otelcol/config.d via root privileges.""" - with tempfile.NamedTemporaryFile( - mode="w", suffix=".yaml", delete=False, encoding="utf-8" - ) as tmp: - tmp.write(config_content) - tmp_path = tmp.name - - try: - mkdir_result = run_as_root( - "mkdir", "-p", CONFIG_DIR # create directory if it doesn't exist - ) - if mkdir_result.returncode != 0: - stderr = mkdir_result.stderr.decode(errors="replace").strip() - logger.error( - "Failed to create collector config directory '%s': %s", - CONFIG_DIR, - stderr or "unknown error", - ) - sys.exit(1) - - install_result = run_as_root( - "install", "-m", "0644", tmp_path, config_path # rw-r--r-- permissions - ) - if install_result.returncode != 0: - stderr = install_result.stderr.decode(errors="replace").strip() - logger.error( - "Failed to install collector config to '%s': %s", - config_path, - stderr or "unknown error", - ) - sys.exit(1) - finally: - os.unlink(tmp_path) - - logger.info("Wrote log-forwarding collector config to: %s", config_path) - - -def restart_collector() -> None: - """Restart collector service so new config is loaded.""" - if SNAP_CMD is None: - logger.error("Required command is missing: snap") - sys.exit(1) - - restart_result = run_as_root(SNAP_CMD, "restart", "opentelemetry-collector") - if restart_result.returncode != 0: - stderr = restart_result.stderr.decode(errors="replace").strip() - logger.error( - "Failed to restart opentelemetry-collector: %s", - stderr or "unknown error", - ) - sys.exit(1) - logger.info("Restarted opentelemetry-collector to apply log-forwarding config.") - - -def main(): - """Validate inputs, write collector config, and restart the collector service.""" - files_input = read_files_input() - config_path = resolve_config_path() - ensure_collector_is_available() - - files = parse_files_into_list(files_input) - if not files: - logger.error("Input 'files' must contain at least one path or glob.") - sys.exit(1) - - resolved_endpoint = resolve_endpoint() - exporter_already_exists = check_exporter_exists() - validate_exporter_configuration(resolved_endpoint, exporter_already_exists) - - config_content = build_config(files, resolved_endpoint, exporter_already_exists) - write_collector_config(config_content, config_path) - restart_collector() - - -if __name__ == "__main__": - main() diff --git a/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py b/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py deleted file mode 100644 index 61404587..00000000 --- a/actions/enable_log_forwarding/tests/test_enable_log_forwarding.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. -"""Unit tests for the enable_log_forwarding action script.""" - -import importlib.util -import json -import pathlib -import unittest -from unittest import mock - -MODULE_PATH = ( - pathlib.Path(__file__).resolve().parent.parent / "enable_log_forwarding.py" -) - - -def load_module(path: pathlib.Path): - """Load the action module from file for direct function-level unit testing.""" - spec = importlib.util.spec_from_file_location("enable_log_forwarding", path) - if spec is None or spec.loader is None: - raise RuntimeError(f"Unable to load module from {path}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -MODULE = load_module(MODULE_PATH) - - -class TestEnableLogForwarding(unittest.TestCase): - """Test suite for parsing, endpoint resolution, exporter detection, and config generation.""" - - def test_parse_files_into_list(self): - """It parses comma/newline-separated inputs and drops empty entries.""" - files_input = " /var/log/a.log,\n/var/log/b.log ,, /var/log/c*.log\n" - parsed = MODULE.parse_files_into_list(files_input) - self.assertEqual( - parsed, ["/var/log/a.log", "/var/log/b.log", "/var/log/c*.log"] - ) - - def test_resolve_endpoint_prefers_input(self): - """It prefers the explicit action input endpoint over fallback env values.""" - with mock.patch.dict( - MODULE.os.environ, - { - "INPUT_OTLP_ENDPOINT": "input-endpoint:4318", - "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT": "system-endpoint:4318", - }, - clear=False, - ): - resolved = MODULE.resolve_endpoint() - - self.assertEqual(resolved, "input-endpoint:4318") - - def test_resolve_endpoint_falls_back_to_action_env(self): - """It falls back to the workflow-provided endpoint when input is empty.""" - with mock.patch.dict( - MODULE.os.environ, - { - "INPUT_OTLP_ENDPOINT": "", - "ACTION_OTEL_EXPORTER_OTLP_ENDPOINT": "system-endpoint:4318", - }, - clear=False, - ): - resolved = MODULE.resolve_endpoint() - - self.assertEqual(resolved, "system-endpoint:4318") - - def test_check_exporter_exists_true(self): - """It returns true when a collector config file defines the exporter name.""" - mock_file = mock.Mock(spec=pathlib.Path) - mock_file.is_file.return_value = True - mock_file.read_text.return_value = "exporters:\n otlp_grpc:\n" - - with mock.patch.object( - pathlib.Path, "is_dir", return_value=True - ), mock.patch.object(pathlib.Path, "rglob", return_value=[mock_file]): - self.assertTrue(MODULE.check_exporter_exists()) - - def test_build_config_adds_exporter_when_missing(self): - """It adds an exporter block when no pre-existing exporter is detected.""" - env = { - "GITHUB_REPOSITORY": "canonical/github-runner-operators", - "RUNNER_NAME": "runner-1", - "GITHUB_WORKFLOW": "CI", - "GITHUB_JOB": "test", - "GITHUB_RUN_ID": "123", - "GITHUB_RUN_ATTEMPT": "1", - } - with mock.patch.dict(MODULE.os.environ, env, clear=False): - raw = MODULE.build_config(["/var/log/syslog"], "otel:4318", False) - - config = json.loads(raw) - self.assertEqual( - config["receivers"]["filelog/github_runner_optin"]["include"], - ["/var/log/syslog"], - ) - self.assertEqual( - config["service"]["pipelines"]["logs/github_runner_optin"]["exporters"], - [MODULE.EXPORTER_NAME], - ) - self.assertEqual( - config["exporters"][MODULE.EXPORTER_NAME]["endpoint"], "otel:4318" - ) - - def test_build_config_reuses_existing_exporter(self): - """It omits exporter creation when an exporter already exists elsewhere.""" - with mock.patch.dict(MODULE.os.environ, {}, clear=False): - raw = MODULE.build_config(["/var/log/syslog"], "otel:4318", True) - - config = json.loads(raw) - self.assertNotIn("exporters", config) - - -if __name__ == "__main__": - unittest.main() diff --git a/docs/conf.py b/docs/conf.py index 9dbfeaf0..274bf1a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -272,6 +272,7 @@ extensions = [ "canonical_sphinx", + "myst_parser", "notfound.extension", "sphinx_design", "sphinx_reredirects", @@ -293,6 +294,11 @@ "sphinx_sitemap", ] +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} + # Excludes files or directories from processing exclude_patterns = [ diff --git a/docs/how-to/enable-log-forwarding.md b/docs/how-to/enable-log-forwarding.md index d7c3faf7..d8549a09 100644 --- a/docs/how-to/enable-log-forwarding.md +++ b/docs/how-to/enable-log-forwarding.md @@ -7,9 +7,12 @@ By default, nothing is forwarded. Log forwarding starts only when this action is ## Prerequisites - Use a self-hosted Linux runner. -- Install the `opentelemetry-collector` snap on the runner. - Ensure the workflow can update `/etc/otelcol/config.d` with root privileges. +When this action runs on a GitHub-hosted runner, it performs a no-op and exits successfully so the job can continue. + +The action installs `opentelemetry-collector` when it is missing. + ## Provide inputs To enable log forwarding, set the following inputs in your workflow file as required by your setup: @@ -18,6 +21,8 @@ To enable log forwarding, set the following inputs in your workflow file as requ - `config-file-name` (optional, default `90-github-runner-log-forwarding.yaml`): generated config file name. - `otlp-endpoint` (optional): OTLP/gRPC endpoint used to create the exporter when one is not already configured. +Avoid adding files that can contain secrets or sensitive information, because forwarded log lines are exported to your telemetry backend. + When `otlp-endpoint` is not set, the action falls back to `ACTION_OTEL_EXPORTER_OTLP_ENDPOINT` from the workflow environment. ## Use the action @@ -29,7 +34,7 @@ jobs: chrony-testing: runs-on: [self-hosted, linux] steps: - - uses: canonical/github-runner-operators/actions/enable_log_forwarding@main + - uses: canonical/github-runner-operators/actions/enable-log-forwarding@main with: files: | /var/log/chrony/*.log @@ -47,9 +52,9 @@ Use these checks to confirm forwarding: The action adds GitHub context as resource attributes on forwarded logs: -- `github.job.id` +- `github.job` - `github.repository` -- `github.runner.name` +- `github.runner` - `github.workflow` - `github.run.id` - `github.run.attempt` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..a9f0bf76 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[tool.mypy] +python_version = "3.12" +ignore_missing_imports = true + +[tool.pylint.main] +jobs = 1 + +[tool.bandit] +exclude_dirs = [ + "actions/enable-log-forwarding/tests", + "charms", + "docs", +] diff --git a/tox.ini b/tox.ini index 70e7628b..b5cce017 100644 --- a/tox.ini +++ b/tox.ini @@ -85,13 +85,18 @@ description = Run static analysis for Python code under actions/ deps = mypy pylint + bandit + pyyaml + types-PyYAML commands = mypy {[vars]actions_path} pylint {[vars]actions_path} + bandit -q -r {[vars]actions_path} -x {[vars]actions_path}/enable-log-forwarding/tests [testenv:actions-unit] description = Run unit tests for Python code under actions/ deps = pytest + pyyaml commands = pytest -v -s {[vars]actions_path} From 9a3491d27dc77e5bda3ee2efe4a01f9e687acc81 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Wed, 29 Apr 2026 11:44:20 +0700 Subject: [PATCH 22/23] remove pyyaml Co-authored-by: Copilot --- actions/enable-log-forwarding/action.yaml | 3 - .../enable_log_forwarding.py | 95 ++++++++++++++++--- .../tests/test_enable_log_forwarding.py | 29 ++++++ tox.ini | 3 - 4 files changed, 109 insertions(+), 21 deletions(-) diff --git a/actions/enable-log-forwarding/action.yaml b/actions/enable-log-forwarding/action.yaml index 007c4ebe..863e4429 100644 --- a/actions/enable-log-forwarding/action.yaml +++ b/actions/enable-log-forwarding/action.yaml @@ -30,9 +30,6 @@ inputs: runs: using: composite steps: - - name: Install action Python dependencies - shell: bash - run: pip install pyyaml --quiet - name: Configure collector for opt-in file log forwarding shell: bash env: diff --git a/actions/enable-log-forwarding/enable_log_forwarding.py b/actions/enable-log-forwarding/enable_log_forwarding.py index c3d49105..255ae9cd 100644 --- a/actions/enable-log-forwarding/enable_log_forwarding.py +++ b/actions/enable-log-forwarding/enable_log_forwarding.py @@ -14,8 +14,6 @@ from pathlib import Path from typing import Sequence -import yaml - CONFIG_DIR = "/etc/otelcol/config.d" EXPORTER_NAME = "otlp_grpc" SNAP_CMD = Path("/usr/bin/snap") @@ -24,6 +22,15 @@ CP_CMD = Path("/usr/bin/cp") CHMOD_CMD = Path("/usr/bin/chmod") FILES_SPLIT_PATTERN = re.compile(r"[,\n]") # character class: split on comma or newline +SUPPORTED_CONFIG_EXTENSIONS = {".yaml", ".yml", ".json"} +# Detects the start of a top-level YAML exporters section (e.g. "exporters:"). +YAML_EXPORTERS_SECTION_PATTERN = re.compile(r"^\s*exporters\s*:\s*(?:#.*)?$") +# Detects a YAML key that matches the exporter name, optionally quoted. +YAML_EXPORTER_KEY_PATTERN_TEMPLATE = r"^\s*['\"]?{exporter_name}['\"]?\s*:\s*(?:#.*)?$" +# Detects a JSON exporters object containing the exporter key. +JSON_EXPORTER_KEY_PATTERN_TEMPLATE = ( + r'"exporters"\s*:\s*\{{[\s\S]*?"{exporter_name}"\s*:' +) logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") logger = logging.getLogger(__name__) @@ -79,30 +86,88 @@ def is_github_hosted_runner() -> bool: return os.getenv("RUNNER_ENVIRONMENT", "").strip().lower() == "github-hosted" +def _contains_exporter_in_yaml(content: str, exporter_name: str) -> bool: + """Return whether YAML content defines exporter_name under exporters: section.""" + in_exporters = False + exporters_indent = -1 + exporter_key_pattern = re.compile( + YAML_EXPORTER_KEY_PATTERN_TEMPLATE.format( + exporter_name=re.escape(exporter_name) + ) + ) + + for line in content.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + + indent = len(line) - len(line.lstrip()) + + if in_exporters: + if indent <= exporters_indent and not stripped.startswith("-"): + in_exporters = False + elif exporter_key_pattern.match(line): + return True + + if YAML_EXPORTERS_SECTION_PATTERN.match(line): + in_exporters = True + exporters_indent = indent + + return False + + +def _contains_exporter_in_json(content: str, exporter_name: str) -> bool: + """Return whether JSON content defines exporter_name inside exporters object.""" + json_pattern = re.compile( + JSON_EXPORTER_KEY_PATTERN_TEMPLATE.format( + exporter_name=re.escape(exporter_name) + ), + ) + return bool(json_pattern.search(content)) + + +def _read_text_file(path: Path) -> str | None: + """Read file content safely, returning None on read errors.""" + try: + return path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None + + +def _is_supported_config_file(path: Path) -> bool: + """Return whether path is a supported config file extension.""" + return path.is_file() and path.suffix.lower() in SUPPORTED_CONFIG_EXTENSIONS + + def exporter_exists_in_config_dir( exporter_name: str, config_dir: str, exclude_path: str ) -> bool: """Check if an exporter with the given name is already defined in another config fragment.""" config_dir_path = Path(config_dir) + exclude = Path(exclude_path).resolve() if not config_dir_path.is_dir(): return False + for config_file in config_dir_path.iterdir(): - if not config_file.is_file() or config_file.suffix not in { - ".yaml", - ".yml", - ".json", - }: + if not _is_supported_config_file(config_file): continue - if str(config_file) == exclude_path: + if config_file.resolve() == exclude: continue - try: - content = yaml.safe_load(config_file.read_text(encoding="utf-8")) - except (yaml.YAMLError, OSError): + + content = _read_text_file(config_file) + if content is None: continue - if isinstance(content, dict): - exporters = content.get("exporters", {}) - if isinstance(exporters, dict) and exporter_name in exporters: - return True + + if config_file.suffix.lower() in { + ".yaml", + ".yml", + } and _contains_exporter_in_yaml(content, exporter_name): + return True + if config_file.suffix.lower() == ".json" and _contains_exporter_in_json( + content, exporter_name + ): + return True + return False diff --git a/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py b/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py index e86a1bd8..d34f0c83 100644 --- a/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py +++ b/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py @@ -215,6 +215,35 @@ def test_exporter_exists_in_config_dir_ignores_exclude_path(): assert found is False +def test_exporter_exists_in_config_dir_finds_exporter_in_json(): + """ + arrange: write a JSON config file containing the fixed exporter name. + act: check whether that exporter exists in the config directory. + assert: returns True when the exporter is present in a JSON fragment. + """ + # Arrange + with tempfile.TemporaryDirectory() as config_dir: + existing = pathlib.Path(config_dir) / "91-other.json" + existing.write_text( + "{\n" + ' "exporters": {\n' + f' "{module.EXPORTER_NAME}": {{\n' + ' "endpoint": "otel:4317"\n' + " }\n" + " }\n" + "}\n" + ) + exclude = str(pathlib.Path(config_dir) / "91-optin.logs.yaml") + + # Act + found = module.exporter_exists_in_config_dir( + module.EXPORTER_NAME, config_dir, exclude + ) + + # Assert + assert found is True + + def test_resolve_endpoint_exits_when_no_endpoint_set(monkeypatch): """ arrange: ensure both endpoint environment variables are unset. diff --git a/tox.ini b/tox.ini index b5cce017..1a568407 100644 --- a/tox.ini +++ b/tox.ini @@ -86,8 +86,6 @@ deps = mypy pylint bandit - pyyaml - types-PyYAML commands = mypy {[vars]actions_path} pylint {[vars]actions_path} @@ -97,6 +95,5 @@ commands = description = Run unit tests for Python code under actions/ deps = pytest - pyyaml commands = pytest -v -s {[vars]actions_path} From 39ae8fba5d3215f0e5842a9bcc358fca61227ec1 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Wed, 29 Apr 2026 13:29:49 +0700 Subject: [PATCH 23/23] parameterized test Co-authored-by: Copilot --- .../tests/test_enable_log_forwarding.py | 238 +++++++++--------- tox.ini | 1 + 2 files changed, 114 insertions(+), 125 deletions(-) diff --git a/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py b/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py index d34f0c83..358b522a 100644 --- a/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py +++ b/actions/enable-log-forwarding/tests/test_enable_log_forwarding.py @@ -9,102 +9,104 @@ import tempfile from typing import Any +import pytest + ACTION_DIR = pathlib.Path(__file__).resolve().parent.parent sys.path.insert(0, str(ACTION_DIR)) module: Any = importlib.import_module("enable_log_forwarding") -def test_parse_files_into_list(): - """ - arrange: prepare comma/newline-separated file input with extra empty tokens. +@pytest.mark.parametrize( + ("files_input", "expected"), + [ + ( + " /var/log/a.log,\n/var/log/b.log ,, /var/log/c*.log\n", + ["/var/log/a.log", "/var/log/b.log", "/var/log/c*.log"], + ), + ( + "\n/var/log/only.log\n", + ["/var/log/only.log"], + ), + ], +) +def test_parse_files_into_list(files_input: str, expected: list[str]): + """ + arrange: provide comma/newline-separated files input variants. act: parse the files input into a normalized list. - assert: parsed output keeps only non-empty, trimmed file paths. + assert: output keeps only non-empty, trimmed file paths. """ # Arrange - files_input = " /var/log/a.log,\n/var/log/b.log ,, /var/log/c*.log\n" # Act parsed = module.parse_files_into_list(files_input) # Assert - assert parsed == ["/var/log/a.log", "/var/log/b.log", "/var/log/c*.log"] - - -def test_resolve_endpoint_prefers_input(monkeypatch): - """ - arrange: set both explicit input and fallback endpoint environment variables. - act: resolve the endpoint used by the action. - assert: explicit input endpoint takes precedence over fallback. - """ - # Arrange - monkeypatch.setenv("INPUT_OTLP_ENDPOINT", "input-endpoint:4318") - monkeypatch.setenv("ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", "system-endpoint:4318") - - # Act - resolved = module.resolve_endpoint() - - # Assert - assert resolved == "input-endpoint:4318" + assert parsed == expected -def test_resolve_endpoint_falls_back_to_action_env(monkeypatch): +@pytest.mark.parametrize( + ("input_endpoint", "fallback_endpoint", "expected_endpoint"), + [ + ("input-endpoint:4318", "system-endpoint:4318", "input-endpoint:4318"), + ("", "system-endpoint:4318", "system-endpoint:4318"), + ], +) +def test_resolve_endpoint_uses_input_then_fallback( + monkeypatch, input_endpoint: str, fallback_endpoint: str, expected_endpoint: str +): """ - arrange: set empty explicit input and a workflow-provided fallback endpoint. + arrange: set explicit input and fallback endpoint environment variables. act: resolve the endpoint used by the action. - assert: fallback workflow endpoint is returned. + assert: resolver returns explicit input first, otherwise fallback endpoint. """ # Arrange - monkeypatch.setenv("INPUT_OTLP_ENDPOINT", "") - monkeypatch.setenv("ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", "system-endpoint:4318") + monkeypatch.setenv("INPUT_OTLP_ENDPOINT", input_endpoint) + monkeypatch.setenv("ACTION_OTEL_EXPORTER_OTLP_ENDPOINT", fallback_endpoint) # Act resolved = module.resolve_endpoint() # Assert - assert resolved == "system-endpoint:4318" - - -def test_is_github_hosted_runner_detects_github_hosted(monkeypatch): - """ - arrange: set RUNNER_ENVIRONMENT to github-hosted. - act: check whether runner is github-hosted. - assert: returns True. - """ - # Arrange - monkeypatch.setenv("RUNNER_ENVIRONMENT", "github-hosted") - - # Act - is_github_hosted = module.is_github_hosted_runner() - - # Assert - assert is_github_hosted is True + assert resolved == expected_endpoint -def test_is_github_hosted_runner_detects_non_github_hosted(monkeypatch): +@pytest.mark.parametrize( + ("runner_environment", "expected"), + [ + ("github-hosted", True), + ("self-hosted", False), + ], +) +def test_is_github_hosted_runner_detection( + monkeypatch, runner_environment: str, expected: bool +): """ - arrange: set RUNNER_ENVIRONMENT to self-hosted. + arrange: set RUNNER_ENVIRONMENT for different runner types. act: check whether runner is github-hosted. - assert: returns False. + assert: returns expected boolean per runner environment. """ # Arrange - monkeypatch.setenv("RUNNER_ENVIRONMENT", "self-hosted") + monkeypatch.setenv("RUNNER_ENVIRONMENT", runner_environment) # Act is_github_hosted = module.is_github_hosted_runner() # Assert - assert is_github_hosted is False + assert is_github_hosted is expected -def test_main_skips_configuration_on_github_hosted(monkeypatch): +@pytest.mark.parametrize("runner_environment", ["github-hosted", "GITHUB-HOSTED"]) +def test_main_skips_configuration_on_github_hosted( + monkeypatch, runner_environment: str +): """ - arrange: set github-hosted environment and guard setup functions. + arrange: set github-hosted environment variants and guard setup functions. act: run main. assert: setup functions are not called and action exits successfully. """ # Arrange - monkeypatch.setenv("RUNNER_ENVIRONMENT", "github-hosted") + monkeypatch.setenv("RUNNER_ENVIRONMENT", runner_environment) def _should_not_be_called() -> str: raise AssertionError("read_files_input should not be called") @@ -115,11 +117,20 @@ def _should_not_be_called() -> str: module.main() -def test_build_config_adds_exporter_when_missing(monkeypatch): +@pytest.mark.parametrize( + ("define_exporter", "has_exporters_block"), + [ + (True, True), + (False, False), + ], +) +def test_build_config_exporter_block_is_conditionally_defined( + monkeypatch, define_exporter: bool, has_exporters_block: bool +): """ arrange: define GitHub metadata env vars. act: build and parse collector config for selected log files. - assert: config includes receiver paths, pipeline exporter, and exporter endpoint. + assert: exporter block is included only when define_exporter is True. """ # Arrange monkeypatch.setenv("GITHUB_REPOSITORY", "canonical/github-runner-operators") @@ -130,7 +141,12 @@ def test_build_config_adds_exporter_when_missing(monkeypatch): monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") # Act - raw = module.build_config(["/var/log/syslog"], "otel:4318", module.EXPORTER_NAME) + raw = module.build_config( + ["/var/log/syslog"], + "otel:4318", + module.EXPORTER_NAME, + define_exporter=define_exporter, + ) config = json.loads(raw) # Assert @@ -140,48 +156,49 @@ def test_build_config_adds_exporter_when_missing(monkeypatch): assert config["service"]["pipelines"]["logs/github_runner_optin"]["exporters"] == [ module.EXPORTER_NAME ] - assert config["exporters"][module.EXPORTER_NAME]["endpoint"] == "otel:4318" - - -def test_build_config_skips_exporter_when_define_exporter_is_false(monkeypatch): - """ - arrange: define GitHub metadata env vars. - act: build config with define_exporter=False. - assert: config has no exporters block but still references the exporter in the pipeline. - """ - # Arrange - monkeypatch.setenv("GITHUB_REPOSITORY", "canonical/github-runner-operators") - monkeypatch.setenv("RUNNER_NAME", "runner-1") - monkeypatch.setenv("GITHUB_WORKFLOW", "CI") - monkeypatch.setenv("GITHUB_JOB", "test") - monkeypatch.setenv("GITHUB_RUN_ID", "123") - monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") - - # Act - raw = module.build_config( - ["/var/log/syslog"], "otel:4318", module.EXPORTER_NAME, define_exporter=False - ) - config = json.loads(raw) - - # Assert - assert "exporters" not in config - assert config["service"]["pipelines"]["logs/github_runner_optin"]["exporters"] == [ - module.EXPORTER_NAME - ] - - -def test_exporter_exists_in_config_dir_finds_exporter(): - """ - arrange: write a YAML config file containing the fixed exporter name. - act: check whether that exporter exists in the config directory. - assert: returns True when the exporter is present in another file. + assert ("exporters" in config) is has_exporters_block + if has_exporters_block: + assert config["exporters"][module.EXPORTER_NAME]["endpoint"] == "otel:4318" + + +@pytest.mark.parametrize( + ("file_name", "content", "expected"), + [ + ( + "91-other.yaml", + f"exporters:\n {module.EXPORTER_NAME}:\n endpoint: otel:4317\n", + True, + ), + ( + "91-other.json", + "{\n" + ' "exporters": {\n' + f' "{module.EXPORTER_NAME}": {{\n' + ' "endpoint": "otel:4317"\n' + " }\n" + " }\n" + "}\n", + True, + ), + ( + "91-other.yaml", + f"receivers:\n {module.EXPORTER_NAME}:\n endpoint: otel:4317\n", + False, + ), + ], +) +def test_exporter_exists_in_config_dir_detects_supported_formats_and_sections( + file_name: str, content: str, expected: bool +): + """ + arrange: prepare config fragments in different formats and sections. + act: check whether exporter exists in the config directory. + assert: returns True only when exporter is defined under exporters in supported formats. """ # Arrange with tempfile.TemporaryDirectory() as config_dir: - existing = pathlib.Path(config_dir) / "91-other.yaml" - existing.write_text( - f"exporters:\n {module.EXPORTER_NAME}:\n endpoint: otel:4317\n" - ) + existing = pathlib.Path(config_dir) / file_name + existing.write_text(content) exclude = str(pathlib.Path(config_dir) / "91-optin.logs.yaml") # Act @@ -190,7 +207,7 @@ def test_exporter_exists_in_config_dir_finds_exporter(): ) # Assert - assert found is True + assert found is expected def test_exporter_exists_in_config_dir_ignores_exclude_path(): @@ -215,35 +232,6 @@ def test_exporter_exists_in_config_dir_ignores_exclude_path(): assert found is False -def test_exporter_exists_in_config_dir_finds_exporter_in_json(): - """ - arrange: write a JSON config file containing the fixed exporter name. - act: check whether that exporter exists in the config directory. - assert: returns True when the exporter is present in a JSON fragment. - """ - # Arrange - with tempfile.TemporaryDirectory() as config_dir: - existing = pathlib.Path(config_dir) / "91-other.json" - existing.write_text( - "{\n" - ' "exporters": {\n' - f' "{module.EXPORTER_NAME}": {{\n' - ' "endpoint": "otel:4317"\n' - " }\n" - " }\n" - "}\n" - ) - exclude = str(pathlib.Path(config_dir) / "91-optin.logs.yaml") - - # Act - found = module.exporter_exists_in_config_dir( - module.EXPORTER_NAME, config_dir, exclude - ) - - # Assert - assert found is True - - def test_resolve_endpoint_exits_when_no_endpoint_set(monkeypatch): """ arrange: ensure both endpoint environment variables are unset. diff --git a/tox.ini b/tox.ini index 1a568407..c8e7294b 100644 --- a/tox.ini +++ b/tox.ini @@ -86,6 +86,7 @@ deps = mypy pylint bandit + pytest commands = mypy {[vars]actions_path} pylint {[vars]actions_path}