From 179c918bdebf174c8bc92323628ac0f110920284 Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Wed, 3 Sep 2025 09:06:10 -0700 Subject: [PATCH 01/13] apply benchmark patch on label --- .github/workflows/apply-benchmark-patch.yml | 83 +++++++++++++++++++++ .github/workflows/run-benchmarks.yml | 10 ++- scripts/benchmark_action.sh | 44 ++++++----- 3 files changed, 116 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/apply-benchmark-patch.yml diff --git a/.github/workflows/apply-benchmark-patch.yml b/.github/workflows/apply-benchmark-patch.yml new file mode 100644 index 0000000000..b7e8d75766 --- /dev/null +++ b/.github/workflows/apply-benchmark-patch.yml @@ -0,0 +1,83 @@ +# .github/workflows/apply-benchmark-patch.yml +name: Apply-Benchmark-Patch + +on: + pull_request: + types: [labeled] + +permissions: + contents: write + pull-requests: write + +jobs: + apply: + if: ${{ github.event.label.name == 'apply-benchmark-patch' }} + runs-on: Benchmarking + + steps: + - name: Check out PR branch + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + - name: Install GitHub CLI + run: | + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" gh + echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token + + - name: Download latest bench patch artifact from failed run + uses: dawidd6/action-download-artifact@v3 + with: + workflow: Validate-Benchmarks + pr: ${{ github.event.pull_request.number }} + workflow_conclusion: failure + name: bench-patch + path: .bench_patch + + - name: Apply and commit patch (if available) + run: | + set -euo pipefail + + if [ ! -f ".bench_patch/benchmark_patch.diff" ]; then + echo "No benchmark patch artifact found for this PR." + echo "You may need to re-run 'Validate-Benchmarks' to generate a fresh patch." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Attempt 3-way apply; if it fails, ask to regenerate + if ! git apply --index --3way .bench_patch/benchmark_patch.diff; then + echo "Patch failed to apply cleanly. Please re-run Validate-Benchmarks to regenerate the patch." + exit 1 + fi + + if git diff --cached --quiet; then + echo "Patch applied but produced no changes (already up to date)." + exit 0 + fi + + echo "==== diff preview ====" + git diff --cached --stat + git diff --cached | head -n 80 || true + echo "======================" + + branch=$(git symbolic-ref --quiet --short HEAD || true) + if [ -z "$branch" ]; then + echo "Not on a branch - cannot push" >&2 + exit 1 + fi + + git commit -m "auto-update benchmark weights" + git push origin "HEAD:${branch}" + + - name: Remove apply-benchmark-patch label + if: ${{ success() }} + run: | + gh pr edit ${{ github.event.pull_request.number }} \ + --repo "${{ github.repository }}" \ + --remove-label "apply-benchmark-patch" || true diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 2e9980532b..75ec1183f8 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -20,7 +20,6 @@ jobs: env: SKIP_BENCHMARKS: "0" - AUTO_COMMIT_WEIGHTS: "1" steps: # ────────────────────────────────────────────────────────────────── @@ -135,6 +134,15 @@ jobs: chmod +x scripts/benchmark_action.sh scripts/benchmark_action.sh + # Persist the prepared patch even if the previous step failed. + - name: Upload patch artifact (if prepared) + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: bench-patch + path: .bench_patch/** + if-no-files-found: ignore + # (6) — final check after run - name: Check skip label after run if: ${{ env.SKIP_BENCHMARKS != '1' }} diff --git a/scripts/benchmark_action.sh b/scripts/benchmark_action.sh index 50b2a3a014..8e2f2103bd 100755 --- a/scripts/benchmark_action.sh +++ b/scripts/benchmark_action.sh @@ -13,10 +13,13 @@ declare -A DISPATCH_PATHS=( THRESHOLD=20 MAX_RETRIES=3 -AUTO_COMMIT="${AUTO_COMMIT_WEIGHTS:-0}" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RUNTIME_WASM="$SCRIPT_DIR/../target/production/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm" +PATCH_DIR="$SCRIPT_DIR/../.bench_patch" +mkdir -p "$PATCH_DIR" + die() { echo "❌ $1" >&2; exit 1; } digits_only() { echo "${1//[^0-9]/}"; } dec() { local d; d=$(digits_only "$1"); echo "$((10#${d:-0}))"; } @@ -62,25 +65,27 @@ patch_reads_writes() { after=$(sha1sum "$4" | cut -d' ' -f1); [[ "$before" != "$after" ]] } -git_commit_and_push() { - local msg="$1" - local branch; branch=$(git symbolic-ref --quiet --short HEAD || true) - [[ -z "$branch" ]] && die "Not on a branch - cannot push" +write_patch_artifacts_and_fail() { + if [ ${#PATCHED_FILES[@]} -eq 0 ]; then + die "Benchmark drift detected but no files were patched." + fi - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" git add "${PATCHED_FILES[@]}" || true - if git diff --cached --quiet; then - echo "ℹ️ No staged changes after patching."; git status --short; return - fi + { + echo "Head SHA: $(git rev-parse HEAD)" + echo + echo "==== Diffstat ====" + git diff --cached --stat || true + } > "$PATCH_DIR/summary.txt" + + git diff --cached --binary > "$PATCH_DIR/benchmark_patch.diff" - echo "==== diff preview ===="; git diff --cached --stat - git diff --cached | head -n 40 || true - echo "======================" + echo "📦 Prepared patch at: $PATCH_DIR/benchmark_patch.diff" + echo "ℹ️ Add the 'apply-benchmark-patch' label to this PR to have CI apply & commit it." - git commit -m "$msg" - git push origin "HEAD:$branch" || die "Push to '${branch}' failed." + # Fail the job so the PR is blocked until the label is set and the apply workflow runs. + exit 2 } echo "Building runtime-benchmarks…" @@ -155,13 +160,13 @@ for pallet in "${PALLET_LIST[@]}"; do done < "$TMP"; flush echo; printf ' %s\n' "${summary[@]}" + (( fail == 0 )) && { echo "✅ '$pallet' within tolerance."; break; } printf ' ❌ %s\n' "${failures[@]}" (( attempt < MAX_RETRIES )) && { echo "→ Retrying …"; (( attempt++ )); continue; } - echo "❌ '$pallet' still failing; patching …" - [[ "$AUTO_COMMIT" != "1" ]] && die "AUTO_COMMIT_WEIGHTS disabled." + echo "❌ '$pallet' still failing; patching (prepare-only) …" changed=0 for fn in $(printf "%s\n" "${!new_weight[@]}" "${!new_reads[@]}" "${!new_writes[@]}" | sort -u); do @@ -179,11 +184,10 @@ for pallet in "${PALLET_LIST[@]}"; do done ################################################################################ -# Commit & push patches +# Commit & push? No — prepare artifact and fail. ################################################################################ if (( ${#PATCHED_FILES[@]} )); then - echo -e "\n📦 Committing patched files …" - git_commit_and_push "auto-update benchmark weights" + write_patch_artifacts_and_fail fi echo -e "\n══════════════════════════════════════" From 6e53d042ea6d9d28de973fce1e01ca1f026babd1 Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:33:30 -0700 Subject: [PATCH 02/13] fix --- .github/workflows/apply-benchmark-patch.yml | 10 ++-- pallets/subtensor/src/macros/dispatches.rs | 6 +- scripts/benchmark_action.sh | 63 +++++++++++++++------ 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/.github/workflows/apply-benchmark-patch.yml b/.github/workflows/apply-benchmark-patch.yml index b7e8d75766..0914a5c6ef 100644 --- a/.github/workflows/apply-benchmark-patch.yml +++ b/.github/workflows/apply-benchmark-patch.yml @@ -8,6 +8,7 @@ on: permissions: contents: write pull-requests: write + actions: read # required to list & download artifacts from another workflow jobs: apply: @@ -31,19 +32,20 @@ jobs: - name: Download latest bench patch artifact from failed run uses: dawidd6/action-download-artifact@v3 with: - workflow: Validate-Benchmarks + workflow: run-benchmarks.yml pr: ${{ github.event.pull_request.number }} workflow_conclusion: failure name: bench-patch path: .bench_patch + allow_forks: true - name: Apply and commit patch (if available) run: | set -euo pipefail if [ ! -f ".bench_patch/benchmark_patch.diff" ]; then - echo "No benchmark patch artifact found for this PR." - echo "You may need to re-run 'Validate-Benchmarks' to generate a fresh patch." + echo "No benchmark_patch.diff found. Either no auto-patch was possible or it wasn't generated." + echo "See .bench_patch/summary.txt for details (if present)." exit 0 fi @@ -52,7 +54,7 @@ jobs: # Attempt 3-way apply; if it fails, ask to regenerate if ! git apply --index --3way .bench_patch/benchmark_patch.diff; then - echo "Patch failed to apply cleanly. Please re-run Validate-Benchmarks to regenerate the patch." + echo "Patch failed to apply cleanly. Re-run Validate-Benchmarks to regenerate the patch." exit 1 fi diff --git a/pallets/subtensor/src/macros/dispatches.rs b/pallets/subtensor/src/macros/dispatches.rs index c842709208..f9ef39d6cf 100644 --- a/pallets/subtensor/src/macros/dispatches.rs +++ b/pallets/subtensor/src/macros/dispatches.rs @@ -2062,9 +2062,9 @@ mod dispatches { /// * `hotkey` (T::AccountId): /// - The hotkey account to designate as the autostake destination. #[pallet::call_index(114)] - #[pallet::weight( - Weight::from_parts(5_170_000, 0).saturating_add(T::DbWeight::get().writes(1_u64)) - )] + #[pallet::weight((Weight::from_parts(5_170_000, 0) + .saturating_add(T::DbWeight::get().reads(0)) + .saturating_add(T::DbWeight::get().writes(1)), DispatchClass::Normal, Pays::No))] pub fn set_coldkey_auto_stake_hotkey( origin: T::RuntimeOrigin, hotkey: T::AccountId, diff --git a/scripts/benchmark_action.sh b/scripts/benchmark_action.sh index 8e2f2103bd..fb898abd37 100755 --- a/scripts/benchmark_action.sh +++ b/scripts/benchmark_action.sh @@ -12,7 +12,7 @@ declare -A DISPATCH_PATHS=( ) THRESHOLD=20 -MAX_RETRIES=3 +MAX_RETRIES=1 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RUNTIME_WASM="$SCRIPT_DIR/../target/production/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm" @@ -35,7 +35,8 @@ patch_weight() { FN="$1" NEWV="$2" perl -0777 -i -pe ' my $n=$ENV{NEWV}; my $hit=0; $hit+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?Weight::from_parts\(\s*)[0-9A-Za-z_]+|$1$n|s; - $hit+=s|(\#\s*\[pallet::weight[^\]]*?Weight::from_parts\(\s*)[0-9A-Za-z_]+(?=[^\]]*?\]\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1$n|s; + # attribute replacement allowing intermediate attributes (e.g., call_index) before pub fn + $hit+=s|(\#\s*\[pallet::weight[^\]]*?Weight::from_parts\(\s*)[0-9A-Za-z_]+(?=[^\]]*\](?:\s*#\[[^\]]*\])*\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1$n|s; END{exit $hit?0:1} ' "$3" || log_warn "patch_weight: no substitution for $1" after=$(sha1sum "$3" | cut -d' ' -f1); [[ "$before" != "$after" ]] @@ -54,37 +55,55 @@ patch_reads_writes() { my $W=$neww eq "" ? $w : u64($neww); return "$pre$R$mid$W$post"; }; - $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?reads_writes\(\s*)([^,]+)(\s*,\s*)([^)]+)|&$rw_sub($1,$2,$3,$4,"")|e; - $h+=s|(\#\s*\[pallet::weight[^\]]*?reads_writes\(\s*)([^,]+)(\s*,\s*)([^)]+)(?=[^\]]*?\]\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|&$rw_sub($1,$2,$3,$4,"")|e; - $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?\.reads\(\s*)([^)]+)|$1.($newr eq "" ? $2 : u64($newr))|e; - $h+=s|(\#\s*\[pallet::weight[^\]]*?\.reads\(\s*)([^)]+)(?=[^\]]*?\]\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1.($newr eq "" ? $2 : u64($newr))|e; - $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?\.writes\(\s*)([^)]+)|$1.($neww eq "" ? $2 : u64($neww))|e; - $h+=s|(\#\s*\[pallet::weight[^\]]*?\.writes\(\s*)([^)]+)(?=[^\]]*?\]\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1.($neww eq "" ? $2 : u64($neww))|e; + # In-body: reads_writes(...) + $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?reads_writes\(\s*)([^,]+)(\s*,\s*)([^)]+)|&$rw_sub($1,$2,$3,$4,"")|sge; + # In-body: .reads(...) and .writes(...) + $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?\.reads\(\s*)([^)]+)|$1.($newr eq "" ? $2 : u64($newr))|sge; + $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?\.writes\(\s*)([^)]+)|$1.($neww eq "" ? $2 : u64($neww))|sge; + + # Attribute: reads_writes(...), tolerate other attributes between ] and pub fn + $h+=s|(\#\s*\[pallet::weight[^\]]*?reads_writes\(\s*)([^,]+)(\s*,\s*)([^)]+)(?=[^\]]*\](?:\s*#\[[^\]]*\])*\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|&$rw_sub($1,$2,$3,$4,"")|sge; + # Attribute: .reads(...) and .writes(...), tolerate other attributes between ] and pub fn + $h+=s|(\#\s*\[pallet::weight[^\]]*?\.reads\(\s*)([^)]+)(?=[^\]]*\](?:\s*#\[[^\]]*\])*\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1.($newr eq "" ? $2 : u64($newr))|sge; + $h+=s|(\#\s*\[pallet::weight[^\]]*?\.writes\(\s*)([^)]+)(?=[^\]]*\](?:\s*#\[[^\]]*\])*\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1.($neww eq "" ? $2 : u64($neww))|sge; + END{exit $h?0:1} ' "$4" || log_warn "patch_reads_writes: no substitution for $1" after=$(sha1sum "$4" | cut -d' ' -f1); [[ "$before" != "$after" ]] } write_patch_artifacts_and_fail() { - if [ ${#PATCHED_FILES[@]} -eq 0 ]; then - die "Benchmark drift detected but no files were patched." - fi - git add "${PATCHED_FILES[@]}" || true { echo "Head SHA: $(git rev-parse HEAD)" echo + echo "==== Benchmark summary ====" + printf '%s\n' "${GLOBAL_SUMMARY[@]}" || true + echo echo "==== Diffstat ====" git diff --cached --stat || true } > "$PATCH_DIR/summary.txt" - git diff --cached --binary > "$PATCH_DIR/benchmark_patch.diff" + git diff --cached --binary > "$PATCH_DIR/benchmark_patch.diff" || true echo "📦 Prepared patch at: $PATCH_DIR/benchmark_patch.diff" echo "ℹ️ Add the 'apply-benchmark-patch' label to this PR to have CI apply & commit it." + exit 2 +} - # Fail the job so the PR is blocked until the label is set and the apply workflow runs. +write_summary_only_and_fail() { + { + echo "Head SHA: $(git rev-parse HEAD)" + echo + echo "==== Benchmark summary ====" + printf '%s\n' "${GLOBAL_SUMMARY[@]}" || true + echo + echo "No auto-patch was generated for the mismatched extrinsics." + echo "Manual update may be required." + } > "$PATCH_DIR/summary.txt" + + echo "⚠️ No patch could be auto-generated. See .bench_patch/summary.txt." exit 2 } @@ -96,6 +115,8 @@ echo " Will benchmark pallets: ${PALLET_LIST[*]}" echo "─────────────────────────────────────────────" PATCHED_FILES=() +GLOBAL_SUMMARY=() +ANY_FAILURE=0 ################################################################################ # Benchmark loop @@ -160,6 +181,7 @@ for pallet in "${PALLET_LIST[@]}"; do done < "$TMP"; flush echo; printf ' %s\n' "${summary[@]}" + GLOBAL_SUMMARY+=("${summary[@]}") (( fail == 0 )) && { echo "✅ '$pallet' within tolerance."; break; } @@ -167,6 +189,7 @@ for pallet in "${PALLET_LIST[@]}"; do (( attempt < MAX_RETRIES )) && { echo "→ Retrying …"; (( attempt++ )); continue; } echo "❌ '$pallet' still failing; patching (prepare-only) …" + ANY_FAILURE=1 changed=0 for fn in $(printf "%s\n" "${!new_weight[@]}" "${!new_reads[@]}" "${!new_writes[@]}" | sort -u); do @@ -184,12 +207,16 @@ for pallet in "${PALLET_LIST[@]}"; do done ################################################################################ -# Commit & push? No — prepare artifact and fail. +# Fail if any mismatch; upload artifacts in workflow step ################################################################################ -if (( ${#PATCHED_FILES[@]} )); then - write_patch_artifacts_and_fail +if (( ANY_FAILURE )); then + if (( ${#PATCHED_FILES[@]} )); then + write_patch_artifacts_and_fail + else + write_summary_only_and_fail + fi fi echo -e "\n══════════════════════════════════════" -echo "All pallets processed ✔" +echo "All pallets processed ✔ (no drift)" echo "══════════════════════════════════════" From 2c2233102a57344ccf31f756978b5ce4bf71c0b2 Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Wed, 3 Sep 2025 12:28:59 -0700 Subject: [PATCH 03/13] test patch --- pallets/subtensor/src/macros/dispatches.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/subtensor/src/macros/dispatches.rs b/pallets/subtensor/src/macros/dispatches.rs index f9ef39d6cf..76046a9094 100644 --- a/pallets/subtensor/src/macros/dispatches.rs +++ b/pallets/subtensor/src/macros/dispatches.rs @@ -2063,7 +2063,7 @@ mod dispatches { /// - The hotkey account to designate as the autostake destination. #[pallet::call_index(114)] #[pallet::weight((Weight::from_parts(5_170_000, 0) - .saturating_add(T::DbWeight::get().reads(0)) + .saturating_add(T::DbWeight::get().reads(1)) .saturating_add(T::DbWeight::get().writes(1)), DispatchClass::Normal, Pays::No))] pub fn set_coldkey_auto_stake_hotkey( origin: T::RuntimeOrigin, From 8dda0f5724cab306b227b8d2d1e4b6f2a4d52de7 Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:44:00 -0700 Subject: [PATCH 04/13] test --- .github/workflows/apply-benchmark-patch.yml | 31 ++++++++++++++------- .github/workflows/run-benchmarks.yml | 2 +- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/apply-benchmark-patch.yml b/.github/workflows/apply-benchmark-patch.yml index 0914a5c6ef..e578de3cbf 100644 --- a/.github/workflows/apply-benchmark-patch.yml +++ b/.github/workflows/apply-benchmark-patch.yml @@ -1,4 +1,3 @@ -# .github/workflows/apply-benchmark-patch.yml name: Apply-Benchmark-Patch on: @@ -8,7 +7,7 @@ on: permissions: contents: write pull-requests: write - actions: read # required to list & download artifacts from another workflow + actions: read # required to list & download artifacts across workflows jobs: apply: @@ -29,32 +28,44 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" gh echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token - - name: Download latest bench patch artifact from failed run + # Look for the most recent run of the heavy workflow for this PR + # that actually produced an artifact named "bench-patch". + - name: Download latest bench patch artifact from heavy workflow uses: dawidd6/action-download-artifact@v3 with: workflow: run-benchmarks.yml pr: ${{ github.event.pull_request.number }} - workflow_conclusion: failure name: bench-patch path: .bench_patch allow_forks: true + check_artifacts: true + search_artifacts: true + workflow_conclusion: "" + if_no_artifact_found: warn - - name: Apply and commit patch (if available) + - name: Apply and commit patch run: | set -euo pipefail + if [ ! -d ".bench_patch" ]; then + echo "No bench-patch artifact directory found." + exit 0 + fi if [ ! -f ".bench_patch/benchmark_patch.diff" ]; then - echo "No benchmark_patch.diff found. Either no auto-patch was possible or it wasn't generated." - echo "See .bench_patch/summary.txt for details (if present)." + echo "No benchmark_patch.diff found (likely no auto-patch possible)." + if [ -f ".bench_patch/summary.txt" ]; then + echo "Summary from heavy workflow:" + sed -n '1,200p' .bench_patch/summary.txt || true + fi exit 0 fi git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - # Attempt 3-way apply; if it fails, ask to regenerate + # Apply with 3-way merge to be resilient to PR changes if ! git apply --index --3way .bench_patch/benchmark_patch.diff; then - echo "Patch failed to apply cleanly. Re-run Validate-Benchmarks to regenerate the patch." + echo "Patch failed to apply cleanly. Please re-run Validate-Benchmarks to regenerate the patch." exit 1 fi @@ -65,7 +76,7 @@ jobs: echo "==== diff preview ====" git diff --cached --stat - git diff --cached | head -n 80 || true + git diff --cached | head -n 100 || true echo "======================" branch=$(git symbolic-ref --quiet --short HEAD || true) diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 75ec1183f8..9245d0d416 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -128,7 +128,7 @@ jobs: uses: nick-fields/retry@v3 with: timeout_minutes: 180 - max_attempts: 3 + max_attempts: 1 retry_wait_seconds: 60 command: | chmod +x scripts/benchmark_action.sh From 34794e50a66f54e5b2560dcde36753574f0ffccc Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Wed, 3 Sep 2025 14:53:14 -0700 Subject: [PATCH 05/13] test --- .github/workflows/apply-benchmark-patch.yml | 8 +++++++- .github/workflows/run-benchmarks.yml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/apply-benchmark-patch.yml b/.github/workflows/apply-benchmark-patch.yml index e578de3cbf..66d960e88d 100644 --- a/.github/workflows/apply-benchmark-patch.yml +++ b/.github/workflows/apply-benchmark-patch.yml @@ -51,6 +51,12 @@ jobs: exit 0 fi + if [ -f ".bench_patch/summary.txt" ]; then + echo "==== summary.txt ====" + sed -n '1,200p' .bench_patch/summary.txt || true + echo "=====================" + fi + if [ ! -f ".bench_patch/benchmark_patch.diff" ]; then echo "No benchmark_patch.diff found (likely no auto-patch possible)." if [ -f ".bench_patch/summary.txt" ]; then @@ -76,7 +82,7 @@ jobs: echo "==== diff preview ====" git diff --cached --stat - git diff --cached | head -n 100 || true + git diff --cached | head -n 120 || true echo "======================" branch=$(git symbolic-ref --quiet --short HEAD || true) diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 9245d0d416..f36085176f 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -140,7 +140,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: bench-patch - path: .bench_patch/** + path: ${{ github.workspace }}/.bench_patch/** if-no-files-found: ignore # (6) — final check after run From 44470bc0f4cb7c2153dd988dccb14f7447f565d3 Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Wed, 3 Sep 2025 16:11:26 -0700 Subject: [PATCH 06/13] test --- .github/workflows/run-benchmarks.yml | 38 +++++++++++++++++----------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index f36085176f..17f3eb237f 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -1,4 +1,3 @@ -# .github/workflows/benchmarks.yml name: Validate-Benchmarks on: @@ -22,7 +21,6 @@ jobs: SKIP_BENCHMARKS: "0" steps: - # ────────────────────────────────────────────────────────────────── - name: Check out PR branch if: ${{ env.SKIP_BENCHMARKS != '1' }} uses: actions/checkout@v4 @@ -123,25 +121,35 @@ jobs: echo "SKIP_BENCHMARKS=1" >> "$GITHUB_ENV" fi - - name: Run & validate benchmarks + - name: Ensure artifact folder exists if: ${{ env.SKIP_BENCHMARKS != '1' }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 180 - max_attempts: 1 - retry_wait_seconds: 60 - command: | - chmod +x scripts/benchmark_action.sh - scripts/benchmark_action.sh - - # Persist the prepared patch even if the previous step failed. + run: mkdir -p .bench_patch + + - name: Run & validate benchmarks (script controls retries) + if: ${{ env.SKIP_BENCHMARKS != '1' }} + timeout-minutes: 180 + run: | + chmod +x scripts/benchmark_action.sh + scripts/benchmark_action.sh + + - name: List artifact contents (for debugging) + if: ${{ always() }} + run: | + echo "Workspace: $GITHUB_WORKSPACE" + if [ -d ".bench_patch" ]; then + echo "== .bench_patch ==" + ls -la .bench_patch || true + else + echo ".bench_patch directory is missing" + fi + - name: Upload patch artifact (if prepared) if: ${{ always() }} uses: actions/upload-artifact@v4 with: name: bench-patch - path: ${{ github.workspace }}/.bench_patch/** - if-no-files-found: ignore + path: ./.bench_patch + # Omit if-no-files-found so we fail loudly if missing unexpectedly # (6) — final check after run - name: Check skip label after run From b4bea14bc973ce6c9f0f783e9c2c496c60550a94 Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Wed, 3 Sep 2025 17:13:37 -0700 Subject: [PATCH 07/13] test --- .github/workflows/apply-benchmark-patch.yml | 34 +++++++++++++-------- .github/workflows/run-benchmarks.yml | 15 +++++++-- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/workflows/apply-benchmark-patch.yml b/.github/workflows/apply-benchmark-patch.yml index 66d960e88d..6f7e94acac 100644 --- a/.github/workflows/apply-benchmark-patch.yml +++ b/.github/workflows/apply-benchmark-patch.yml @@ -1,3 +1,4 @@ +# .github/workflows/apply-benchmark-patch.yml name: Apply-Benchmark-Patch on: @@ -28,26 +29,38 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" gh echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token - # Look for the most recent run of the heavy workflow for this PR - # that actually produced an artifact named "bench-patch". - name: Download latest bench patch artifact from heavy workflow uses: dawidd6/action-download-artifact@v3 with: - workflow: run-benchmarks.yml + workflow: run-benchmarks.yml # heavy workflow FILE name pr: ${{ github.event.pull_request.number }} name: bench-patch - path: .bench_patch + path: . allow_forks: true check_artifacts: true search_artifacts: true - workflow_conclusion: "" + workflow_conclusion: "" # accept any conclusion if_no_artifact_found: warn + - name: Extract bench patch archive + run: | + set -euo pipefail + if [ -f "bench-patch.tgz" ]; then + tar -xzf bench-patch.tgz + elif [ -f "bench-patch/bench-patch.tgz" ]; then + tar -xzf bench-patch/bench-patch.tgz + else + echo "No bench-patch.tgz found after download." + exit 0 + fi + ls -la .bench_patch || true + - name: Apply and commit patch run: | set -euo pipefail + if [ ! -d ".bench_patch" ]; then - echo "No bench-patch artifact directory found." + echo "No .bench_patch directory found after extraction." exit 0 fi @@ -58,20 +71,15 @@ jobs: fi if [ ! -f ".bench_patch/benchmark_patch.diff" ]; then - echo "No benchmark_patch.diff found (likely no auto-patch possible)." - if [ -f ".bench_patch/summary.txt" ]; then - echo "Summary from heavy workflow:" - sed -n '1,200p' .bench_patch/summary.txt || true - fi + echo "No benchmark_patch.diff found (no auto-patch created)." exit 0 fi git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - # Apply with 3-way merge to be resilient to PR changes if ! git apply --index --3way .bench_patch/benchmark_patch.diff; then - echo "Patch failed to apply cleanly. Please re-run Validate-Benchmarks to regenerate the patch." + echo "Patch failed to apply cleanly. Please re-run Validate-Benchmarks to regenerate." exit 1 fi diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 17f3eb237f..9063996046 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -1,3 +1,4 @@ +# .github/workflows/run-benchmarks.yml name: Validate-Benchmarks on: @@ -143,13 +144,23 @@ jobs: echo ".bench_patch directory is missing" fi + - name: Archive bench patch + if: ${{ always() }} + run: | + if [ -d ".bench_patch" ]; then + tar -czf bench-patch.tgz .bench_patch + ls -lh bench-patch.tgz + else + echo "No .bench_patch directory to archive." + fi + - name: Upload patch artifact (if prepared) if: ${{ always() }} uses: actions/upload-artifact@v4 with: name: bench-patch - path: ./.bench_patch - # Omit if-no-files-found so we fail loudly if missing unexpectedly + path: bench-patch.tgz + if-no-files-found: warn # (6) — final check after run - name: Check skip label after run From 87cf264e2e55ba8ccc2e7ec7e676490975764a1e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Sep 2025 01:41:23 +0000 Subject: [PATCH 08/13] auto-update benchmark weights --- pallets/subtensor/src/macros/dispatches.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pallets/subtensor/src/macros/dispatches.rs b/pallets/subtensor/src/macros/dispatches.rs index 76046a9094..29654869f1 100644 --- a/pallets/subtensor/src/macros/dispatches.rs +++ b/pallets/subtensor/src/macros/dispatches.rs @@ -665,7 +665,7 @@ mod dispatches { /// - Attempting to set prometheus information withing the rate limit min. /// #[pallet::call_index(40)] - #[pallet::weight((Weight::from_parts(32_310_000, 0) + #[pallet::weight((Weight::from_parts(41_240_000, 0) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)), DispatchClass::Normal, Pays::No))] pub fn serve_axon_tls( @@ -2063,7 +2063,7 @@ mod dispatches { /// - The hotkey account to designate as the autostake destination. #[pallet::call_index(114)] #[pallet::weight((Weight::from_parts(5_170_000, 0) - .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().reads(0_u64)) .saturating_add(T::DbWeight::get().writes(1)), DispatchClass::Normal, Pays::No))] pub fn set_coldkey_auto_stake_hotkey( origin: T::RuntimeOrigin, From 31bc038e2b9d2d92518fef4542ca9453dd019b0f Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Wed, 3 Sep 2025 18:53:03 -0700 Subject: [PATCH 09/13] reset --- .github/workflows/apply-benchmark-patch.yml | 4 ++-- scripts/benchmark_action.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/apply-benchmark-patch.yml b/.github/workflows/apply-benchmark-patch.yml index 6f7e94acac..16b499bea6 100644 --- a/.github/workflows/apply-benchmark-patch.yml +++ b/.github/workflows/apply-benchmark-patch.yml @@ -32,14 +32,14 @@ jobs: - name: Download latest bench patch artifact from heavy workflow uses: dawidd6/action-download-artifact@v3 with: - workflow: run-benchmarks.yml # heavy workflow FILE name + workflow: run-benchmarks.yml pr: ${{ github.event.pull_request.number }} name: bench-patch path: . allow_forks: true check_artifacts: true search_artifacts: true - workflow_conclusion: "" # accept any conclusion + workflow_conclusion: "" if_no_artifact_found: warn - name: Extract bench patch archive diff --git a/scripts/benchmark_action.sh b/scripts/benchmark_action.sh index fb898abd37..78c38d7664 100755 --- a/scripts/benchmark_action.sh +++ b/scripts/benchmark_action.sh @@ -12,7 +12,7 @@ declare -A DISPATCH_PATHS=( ) THRESHOLD=20 -MAX_RETRIES=1 +MAX_RETRIES=3 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RUNTIME_WASM="$SCRIPT_DIR/../target/production/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm" From dfd390b0933cf47a5ce38ddca11283dee9c7a955 Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Thu, 4 Sep 2025 07:39:24 -0700 Subject: [PATCH 10/13] final test --- pallets/admin-utils/src/lib.rs | 2 +- scripts/benchmark_action.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pallets/admin-utils/src/lib.rs b/pallets/admin-utils/src/lib.rs index 1b0407f312..e92139d047 100644 --- a/pallets/admin-utils/src/lib.rs +++ b/pallets/admin-utils/src/lib.rs @@ -1598,7 +1598,7 @@ pub mod pallet { /// Sets the number of immune owner neurons #[pallet::call_index(72)] #[pallet::weight(Weight::from_parts(15_000_000, 0) - .saturating_add(::DbWeight::get().reads(1_u64)) + .saturating_add(::DbWeight::get().reads(0_u64)) .saturating_add(::DbWeight::get().writes(1_u64)))] pub fn sudo_set_owner_immune_neuron_limit( origin: OriginFor, diff --git a/scripts/benchmark_action.sh b/scripts/benchmark_action.sh index 78c38d7664..105cbe9bf5 100755 --- a/scripts/benchmark_action.sh +++ b/scripts/benchmark_action.sh @@ -218,5 +218,5 @@ if (( ANY_FAILURE )); then fi echo -e "\n══════════════════════════════════════" -echo "All pallets processed ✔ (no drift)" +echo "All pallets processed ✔" echo "══════════════════════════════════════" From 48d88d9f99ee2fb346993a0a0f005a21eaa2dc53 Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Thu, 4 Sep 2025 08:40:25 -0700 Subject: [PATCH 11/13] add sudo_set_owner_immune_neuron_limit --- .github/workflows/run-benchmarks.yml | 2 +- pallets/admin-utils/src/benchmarking.rs | 11 +++++++++++ pallets/admin-utils/src/lib.rs | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 9063996046..12df3f8ab6 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -126,7 +126,7 @@ jobs: if: ${{ env.SKIP_BENCHMARKS != '1' }} run: mkdir -p .bench_patch - - name: Run & validate benchmarks (script controls retries) + - name: Run & validate benchmarks if: ${{ env.SKIP_BENCHMARKS != '1' }} timeout-minutes: 180 run: | diff --git a/pallets/admin-utils/src/benchmarking.rs b/pallets/admin-utils/src/benchmarking.rs index 4d2ac21045..c824a879a5 100644 --- a/pallets/admin-utils/src/benchmarking.rs +++ b/pallets/admin-utils/src/benchmarking.rs @@ -346,5 +346,16 @@ mod benchmarks { _(RawOrigin::Root, 5u16/*version*/)/*sudo_set_commit_reveal_version()*/; } + #[benchmark] + fn sudo_set_owner_immune_neuron_limit() { + pallet_subtensor::Pallet::::init_new_network( + 1u16.into(), /*netuid*/ + 1u16, /*sudo_tempo*/ + ); + + #[extrinsic_call] + _(RawOrigin::Root, 1u16.into()/*netuid*/, 5u16/*immune_neurons*/)/*sudo_set_owner_immune_neuron_limit()*/; + } + //impl_benchmark_test_suite!(AdminUtils, crate::mock::new_test_ext(), crate::mock::Test); } diff --git a/pallets/admin-utils/src/lib.rs b/pallets/admin-utils/src/lib.rs index e92139d047..95fe4533e6 100644 --- a/pallets/admin-utils/src/lib.rs +++ b/pallets/admin-utils/src/lib.rs @@ -1599,7 +1599,7 @@ pub mod pallet { #[pallet::call_index(72)] #[pallet::weight(Weight::from_parts(15_000_000, 0) .saturating_add(::DbWeight::get().reads(0_u64)) - .saturating_add(::DbWeight::get().writes(1_u64)))] + .saturating_add(::DbWeight::get().writes(0_u64)))] pub fn sudo_set_owner_immune_neuron_limit( origin: OriginFor, netuid: NetUid, From 786773b7579365f9fd619d5461f493fa080c0cb0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 4 Sep 2025 16:49:33 +0000 Subject: [PATCH 12/13] auto-update benchmark weights --- pallets/admin-utils/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pallets/admin-utils/src/lib.rs b/pallets/admin-utils/src/lib.rs index 95fe4533e6..075b8e1976 100644 --- a/pallets/admin-utils/src/lib.rs +++ b/pallets/admin-utils/src/lib.rs @@ -690,7 +690,7 @@ pub mod pallet { /// It is only callable by root and subnet owner. /// The extrinsic will call the Subtensor pallet to set the maximum burn. #[pallet::call_index(23)] - #[pallet::weight(Weight::from_parts(15_940_000, 0) + #[pallet::weight(Weight::from_parts(19_420_000, 0) .saturating_add(::DbWeight::get().reads(2_u64)) .saturating_add(::DbWeight::get().writes(1_u64)))] pub fn sudo_set_max_burn( @@ -1597,9 +1597,9 @@ pub mod pallet { /// Sets the number of immune owner neurons #[pallet::call_index(72)] - #[pallet::weight(Weight::from_parts(15_000_000, 0) + #[pallet::weight(Weight::from_parts(4_639_000, 0) .saturating_add(::DbWeight::get().reads(0_u64)) - .saturating_add(::DbWeight::get().writes(0_u64)))] + .saturating_add(::DbWeight::get().writes(1_u64)))] pub fn sudo_set_owner_immune_neuron_limit( origin: OriginFor, netuid: NetUid, From 2d6e4703a93a92fd203e24534ba77f442a07c5ee Mon Sep 17 00:00:00 2001 From: John Reed <87283488+JohnReedV@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:26:19 -0700 Subject: [PATCH 13/13] blank commit