From 93cf4e6793795aeea577317653f9f970387591d8 Mon Sep 17 00:00:00 2001 From: Juan Diaz Suarez Date: Thu, 16 Apr 2026 11:22:27 +0200 Subject: [PATCH 1/5] feat: add update-refs skill --- skills/update-refs/SKILL.md | 79 ++++++++++++ skills/update-refs/scripts/update_refs.sh | 140 ++++++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 skills/update-refs/SKILL.md create mode 100755 skills/update-refs/scripts/update_refs.sh diff --git a/skills/update-refs/SKILL.md b/skills/update-refs/SKILL.md new file mode 100644 index 0000000..868e311 --- /dev/null +++ b/skills/update-refs/SKILL.md @@ -0,0 +1,79 @@ +--- +name: update-refs +description: >- + Update SaaS file refs in app-interface to the latest + commit SHAs. Use when the user says "update refs", + "bump refs", "promote", or asks to update + ccx-data-pipeline service references. +--- + +# update-refs + +Updates `ref:` fields in ccx-data-pipeline SaaS YAML files +(in app-interface) to the latest commit SHA from each repo's +default branch. Skips `ref: internal`, `main`, and `master`. + +## Prerequisites + +- Git access to `https://gitlab.cee.redhat.com/service/app-interface.git` +- Git access to the GitHub repos whose refs are being updated +- Bash 4+ (for associative arrays) + +## Usage + +Run the script from the `skills/update-refs` directory: + +```bash +./scripts/update_refs.sh [--dry-run] [--repo ]... +``` + +### Options + +| Flag | Description | +|------|-------------| +| `--dry-run` | Show what would change without modifying files | +| `--repo ` | Only update refs for this repo (repeatable). Accepts a full URL or just the repo name. Uses **exact match** — `insights-results-aggregator` will not match `insights-results-aggregator-cleaner`. | + +### Examples + +```bash +# Dry-run for all repos +./scripts/update_refs.sh --dry-run + +# Update a single repo +./scripts/update_refs.sh --repo insights-results-aggregator + +# Update two specific repos +./scripts/update_refs.sh --repo insights-results-aggregator --repo ccx-notification-writer + +# Full URL also works +./scripts/update_refs.sh --repo https://github.com/RedHatInsights/insights-results-aggregator +``` + +## Workflow + +1. **Ask the user** which repos to update, or whether to update all. + Always start with `--dry-run` so the user can review changes. +2. Run the script with `--dry-run` and present the output. +3. After user confirmation, run without `--dry-run`. +4. The script modifies files inside the local app-interface + clone at `/tmp/app-interface`. The user can then `cd` there + to review and submit a merge request. + +## How it works + +1. Clones (or reuses) app-interface to `/tmp/app-interface`. But we always + clone in /tmp in order not to create conflicts with the user's local + app-interface clone. +2. Scans all YAML files under `data/services/insights/ccx-data-pipeline`. +3. For each `url:` + `ref:` pair, fetches the latest SHA from + the repo's default branch via `git ls-remote`. +4. Replaces outdated SHA refs in-place (or reports them in dry-run). + +## Constraints + +- **Always dry-run first** — never modify files without user review. +- **Do not touch branch refs** — `main`, `master`, and `internal` + are intentionally skipped. +- The script requires VPN network access to both GitLab (app-interface) + and GitHub (source repos). diff --git a/skills/update-refs/scripts/update_refs.sh b/skills/update-refs/scripts/update_refs.sh new file mode 100755 index 0000000..a76159e --- /dev/null +++ b/skills/update-refs/scripts/update_refs.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Update all GitHub/GitLab refs in ccx-data-pipeline saas files to latest commit SHAs. +# Skips ref: internal (ephemeral/bonfire targets), main and master. +# Usage: update_refs.sh [--dry-run] [--repo ]... +# --repo filters by exact repo name or full URL (can be repeated). +set -uo pipefail + +PATH_TO_APP_INTERFACE="" # if empty, it will clone a copy in /tmp +if [[ -z "$PATH_TO_APP_INTERFACE" ]]; then + PATH_TO_APP_INTERFACE="/tmp/app-interface" + if [[ -d "$PATH_TO_APP_INTERFACE/.git" ]]; then + echo "App-interface repo already exists at $PATH_TO_APP_INTERFACE, skipping clone." + else + git clone --depth 1 https://gitlab.cee.redhat.com/service/app-interface.git "$PATH_TO_APP_INTERFACE" + fi +fi + +cd "$PATH_TO_APP_INTERFACE" + +git fetch origin master +git checkout master +git pull origin master + +BASE_DIR="data/services/insights/ccx-data-pipeline" +declare -A SHA_CACHE=() # repo_url -> latest_sha +declare -A BRANCH_CACHE=() # repo_url -> default_branch +declare -a REPO_FILTERS=() # exact repo URLs/names to update +DRY_RUN=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true; echo "=== DRY RUN ===" ;; + --repo) + [[ -z "${2:-}" ]] && echo "ERROR: --repo requires a value" >&2 && exit 1 + REPO_FILTERS+=("$2"); shift ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac + shift +done + +repo_matches() { + local url="$1" + [[ ${#REPO_FILTERS[@]} -eq 0 ]] && return 0 + local repo_name="${url##*/}" + for filter in "${REPO_FILTERS[@]}"; do + [[ "$url" == "$filter" || "$repo_name" == "$filter" ]] && return 0 + done + return 1 +} + +get_default_branch() { + local url="$1" + if [[ -n "${BRANCH_CACHE[$url]:-}" ]]; then + REPLY="${BRANCH_CACHE[$url]}" + return + fi + REPLY=$(git ls-remote --symref "$url" HEAD 2>/dev/null \ + | awk '/^ref:/{sub("ref: refs/heads/",""); print $1; exit}') + [[ -z "$REPLY" ]] && REPLY="main" + BRANCH_CACHE["$url"]="$REPLY" +} + +get_latest_sha() { + local url="$1" + if [[ -n "${SHA_CACHE[$url]:-}" ]]; then + REPLY="${SHA_CACHE[$url]}" + return + fi + get_default_branch "$url" + local branch="$REPLY" + REPLY=$(git ls-remote "$url" "refs/heads/$branch" 2>/dev/null | awk '{print $1}') + if [[ -z "$REPLY" ]]; then + echo "WARNING: Could not fetch SHA for $url ($branch)" >&2 + return 1 + fi + SHA_CACHE["$url"]="$REPLY" +} + +process_file() { + local file="$1" + local current_url="" changes=0 + local tmpfile + tmpfile=$(mktemp) + + while IFS= read -r line; do + # Track current url context + if [[ "$line" =~ ^[[:space:]]+url:[[:space:]]+(https://[^[:space:]]+) ]]; then + current_url="${BASH_REMATCH[1]}" + fi + + # Match ref lines, skip 'internal' + if [[ "$line" =~ ^([[:space:]]+ref:[[:space:]]+)([^[:space:]]+)$ ]]; then + local prefix="${BASH_REMATCH[1]}" + local old_ref="${BASH_REMATCH[2]}" + + if [[ "$old_ref" != "internal" && "$old_ref" != "main" && "$old_ref" != "master" && -n "$current_url" ]] && repo_matches "$current_url"; then + if get_latest_sha "$current_url"; then + local new_sha="$REPLY" + if [[ "$old_ref" != "$new_sha" ]]; then + echo " $old_ref -> ${new_sha:0:12}... ($current_url)" >&2 + line="${prefix}${new_sha}" + ((changes++)) || true + fi + fi + fi + fi + printf '%s\n' "$line" + done < "$file" > "$tmpfile" + + if (( changes > 0 )); then + echo "[$file] $changes ref(s) updated" + if [[ "$DRY_RUN" == false ]]; then + mv "$tmpfile" "$file" + else + rm "$tmpfile" + fi + else + rm "$tmpfile" + fi +} + +if [[ ${#REPO_FILTERS[@]} -gt 0 ]]; then + echo "Filtering repos: ${REPO_FILTERS[*]}" +fi + +echo "Collecting YAML files from $BASE_DIR ..." +mapfile -t files < <(find "$BASE_DIR" -type f \( -name '*.yml' -o -name '*.yaml' \) | sort) + +echo "Found ${#files[@]} YAML files. Scanning for refs..." +echo + +for f in "${files[@]}"; do + # Quick check: skip files without both url: and ref: + if grep -qE '^\s+url:\s+https://' "$f" && grep -qE '^\s+ref:\s' "$f"; then + process_file "$f" + fi +done + +echo +echo "Done. ${#SHA_CACHE[@]} unique repo(s) processed." From 548d08a7483eb4119da75fb7a871dabd54fadea5 Mon Sep 17 00:00:00 2001 From: Juan Diaz Suarez Date: Thu, 16 Apr 2026 12:14:24 +0200 Subject: [PATCH 2/5] feat: add update-refs skill --- skills/update-refs/SKILL.md | 26 +++++++++++--- skills/update-refs/scripts/update_refs.sh | 41 +++++++++++++---------- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/skills/update-refs/SKILL.md b/skills/update-refs/SKILL.md index 868e311..3da0d5a 100644 --- a/skills/update-refs/SKILL.md +++ b/skills/update-refs/SKILL.md @@ -24,7 +24,7 @@ default branch. Skips `ref: internal`, `main`, and `master`. Run the script from the `skills/update-refs` directory: ```bash -./scripts/update_refs.sh [--dry-run] [--repo ]... +./scripts/update_refs.sh [--dry-run] [--repo ]... [--local-folder ] ``` ### Options @@ -33,6 +33,7 @@ Run the script from the `skills/update-refs` directory: |------|-------------| | `--dry-run` | Show what would change without modifying files | | `--repo ` | Only update refs for this repo (repeatable). Accepts a full URL or just the repo name. Uses **exact match** — `insights-results-aggregator` will not match `insights-results-aggregator-cleaner`. | +| `--local-folder ` | Use an existing local app-interface checkout instead of cloning to `/tmp/app-interface`. | ### Examples @@ -54,11 +55,28 @@ Run the script from the `skills/update-refs` directory: 1. **Ask the user** which repos to update, or whether to update all. Always start with `--dry-run` so the user can review changes. -2. Run the script with `--dry-run` and present the output. -3. After user confirmation, run without `--dry-run`. -4. The script modifies files inside the local app-interface +2. **Ask the user** if you should clone the app-interface repository or use + the local one. If so, use `--local-folder` option. +3. Run the script with `--dry-run` and present the output. +4. After user confirmation, run without `--dry-run`. +5. The script modifies files inside the local app-interface clone at `/tmp/app-interface`. The user can then `cd` there to review and submit a merge request. +6. Checkout to a branch named `update-refs-` and push the changes. + If not using `--local-folder`, **ask the user** for the fork: +```bash +BRANCH="update-refs-$(date +%Y%m%d)" +git checkout -b $BRANCH +git add . +git commit -m "chore: update refs" +git push -o merge_request.create \ + -o merge_request.remove_source_branch \ + -o merge_request.target=master \ + -o merge_request.title="chore: update refs" \ + fork ${BRANCH} --verbose +``` +1. Tell the user to follow the merge request CI in order + to ask the rest of the team to review the changes. ## How it works diff --git a/skills/update-refs/scripts/update_refs.sh b/skills/update-refs/scripts/update_refs.sh index a76159e..6c84599 100755 --- a/skills/update-refs/scripts/update_refs.sh +++ b/skills/update-refs/scripts/update_refs.sh @@ -1,31 +1,16 @@ #!/usr/bin/env bash # Update all GitHub/GitLab refs in ccx-data-pipeline saas files to latest commit SHAs. # Skips ref: internal (ephemeral/bonfire targets), main and master. -# Usage: update_refs.sh [--dry-run] [--repo ]... +# Usage: update_refs.sh [--dry-run] [--repo ]... [--local-folder ] # --repo filters by exact repo name or full URL (can be repeated). +# --local-folder uses an existing app-interface checkout instead of cloning. set -uo pipefail -PATH_TO_APP_INTERFACE="" # if empty, it will clone a copy in /tmp -if [[ -z "$PATH_TO_APP_INTERFACE" ]]; then - PATH_TO_APP_INTERFACE="/tmp/app-interface" - if [[ -d "$PATH_TO_APP_INTERFACE/.git" ]]; then - echo "App-interface repo already exists at $PATH_TO_APP_INTERFACE, skipping clone." - else - git clone --depth 1 https://gitlab.cee.redhat.com/service/app-interface.git "$PATH_TO_APP_INTERFACE" - fi -fi - -cd "$PATH_TO_APP_INTERFACE" - -git fetch origin master -git checkout master -git pull origin master - -BASE_DIR="data/services/insights/ccx-data-pipeline" declare -A SHA_CACHE=() # repo_url -> latest_sha declare -A BRANCH_CACHE=() # repo_url -> default_branch declare -a REPO_FILTERS=() # exact repo URLs/names to update DRY_RUN=false +PATH_TO_APP_INTERFACE="" while [[ $# -gt 0 ]]; do case "$1" in @@ -33,11 +18,31 @@ while [[ $# -gt 0 ]]; do --repo) [[ -z "${2:-}" ]] && echo "ERROR: --repo requires a value" >&2 && exit 1 REPO_FILTERS+=("$2"); shift ;; + --local-folder) + [[ -z "${2:-}" ]] && echo "ERROR: --local-folder requires a path" >&2 && exit 1 + PATH_TO_APP_INTERFACE="$2"; shift ;; *) echo "Unknown option: $1" >&2; exit 1 ;; esac shift done +if [[ -z "$PATH_TO_APP_INTERFACE" ]]; then + PATH_TO_APP_INTERFACE="/tmp/app-interface" + if [[ -d "$PATH_TO_APP_INTERFACE/.git" ]]; then + echo "App-interface repo already exists at $PATH_TO_APP_INTERFACE, skipping clone." + else + git clone --depth 1 git@gitlab.cee.redhat.com:service/app-interface.git "$PATH_TO_APP_INTERFACE" + fi +fi + +cd "$PATH_TO_APP_INTERFACE" + +git fetch origin master +git checkout master +git pull origin master + +BASE_DIR="data/services/insights/ccx-data-pipeline" + repo_matches() { local url="$1" [[ ${#REPO_FILTERS[@]} -eq 0 ]] && return 0 From d1ada66a6991cafe968fad122a44a6322203d9fb Mon Sep 17 00:00:00 2001 From: Juan Diaz Suarez Date: Thu, 16 Apr 2026 12:15:03 +0200 Subject: [PATCH 3/5] feat: add update-refs skill --- skills/update-refs/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/update-refs/SKILL.md b/skills/update-refs/SKILL.md index 3da0d5a..e135b0e 100644 --- a/skills/update-refs/SKILL.md +++ b/skills/update-refs/SKILL.md @@ -67,7 +67,7 @@ Run the script from the `skills/update-refs` directory: ```bash BRANCH="update-refs-$(date +%Y%m%d)" git checkout -b $BRANCH -git add . +git add data/services/insights/ccx-data-pipeline git commit -m "chore: update refs" git push -o merge_request.create \ -o merge_request.remove_source_branch \ From 41e98a170903eedad83b0dc02cc345e63ef68a27 Mon Sep 17 00:00:00 2001 From: Juan Diaz Suarez Date: Mon, 20 Apr 2026 14:55:10 +0200 Subject: [PATCH 4/5] refactor: show only 7 chars --- skills/update-refs/scripts/update_refs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/update-refs/scripts/update_refs.sh b/skills/update-refs/scripts/update_refs.sh index 6c84599..9c7f8d1 100755 --- a/skills/update-refs/scripts/update_refs.sh +++ b/skills/update-refs/scripts/update_refs.sh @@ -102,7 +102,7 @@ process_file() { if get_latest_sha "$current_url"; then local new_sha="$REPLY" if [[ "$old_ref" != "$new_sha" ]]; then - echo " $old_ref -> ${new_sha:0:12}... ($current_url)" >&2 + echo " $old_ref -> ${new_sha:0:7}... ($current_url)" >&2 line="${prefix}${new_sha}" ((changes++)) || true fi From a7d1ca6d5915fefb5d0f445f9ff776101d3aa9b4 Mon Sep 17 00:00:00 2001 From: Juan Diaz Suarez Date: Wed, 22 Apr 2026 16:12:30 +0200 Subject: [PATCH 5/5] fix: lint --- skills/update-refs/scripts/update_refs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/update-refs/scripts/update_refs.sh b/skills/update-refs/scripts/update_refs.sh index 9c7f8d1..a47eb43 100755 --- a/skills/update-refs/scripts/update_refs.sh +++ b/skills/update-refs/scripts/update_refs.sh @@ -35,7 +35,7 @@ if [[ -z "$PATH_TO_APP_INTERFACE" ]]; then fi fi -cd "$PATH_TO_APP_INTERFACE" +cd "$PATH_TO_APP_INTERFACE" || exit 1 git fetch origin master git checkout master