diff --git a/.github/workflows/apply-benchmark-patch.yml b/.github/workflows/apply-benchmark-patch.yml new file mode 100644 index 0000000000..16b499bea6 --- /dev/null +++ b/.github/workflows/apply-benchmark-patch.yml @@ -0,0 +1,110 @@ +# .github/workflows/apply-benchmark-patch.yml +name: Apply-Benchmark-Patch + +on: + pull_request: + types: [labeled] + +permissions: + contents: write + pull-requests: write + actions: read # required to list & download artifacts across workflows + +jobs: + apply: + if: ${{ github.event.label.name == 'apply-benchmark-patch' }} + runs-on: Benchmarking + + steps: + - name: Check out PR branch + uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + + - name: Install GitHub CLI + run: | + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get update + sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a apt-get install -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" gh + echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token + + - name: Download latest bench patch artifact from heavy workflow + uses: dawidd6/action-download-artifact@v3 + with: + workflow: run-benchmarks.yml + pr: ${{ github.event.pull_request.number }} + name: bench-patch + path: . + allow_forks: true + check_artifacts: true + search_artifacts: true + workflow_conclusion: "" + if_no_artifact_found: warn + + - name: Extract bench patch archive + run: | + set -euo pipefail + if [ -f "bench-patch.tgz" ]; then + tar -xzf bench-patch.tgz + elif [ -f "bench-patch/bench-patch.tgz" ]; then + tar -xzf bench-patch/bench-patch.tgz + else + echo "No bench-patch.tgz found after download." + exit 0 + fi + ls -la .bench_patch || true + + - name: Apply and commit patch + run: | + set -euo pipefail + + if [ ! -d ".bench_patch" ]; then + echo "No .bench_patch directory found after extraction." + exit 0 + fi + + if [ -f ".bench_patch/summary.txt" ]; then + echo "==== summary.txt ====" + sed -n '1,200p' .bench_patch/summary.txt || true + echo "=====================" + fi + + if [ ! -f ".bench_patch/benchmark_patch.diff" ]; then + echo "No benchmark_patch.diff found (no auto-patch created)." + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + if ! git apply --index --3way .bench_patch/benchmark_patch.diff; then + echo "Patch failed to apply cleanly. Please re-run Validate-Benchmarks to regenerate." + exit 1 + fi + + if git diff --cached --quiet; then + echo "Patch applied but produced no changes (already up to date)." + exit 0 + fi + + echo "==== diff preview ====" + git diff --cached --stat + git diff --cached | head -n 120 || true + echo "======================" + + branch=$(git symbolic-ref --quiet --short HEAD || true) + if [ -z "$branch" ]; then + echo "Not on a branch - cannot push" >&2 + exit 1 + fi + + git commit -m "auto-update benchmark weights" + git push origin "HEAD:${branch}" + + - name: Remove apply-benchmark-patch label + if: ${{ success() }} + run: | + gh pr edit ${{ github.event.pull_request.number }} \ + --repo "${{ github.repository }}" \ + --remove-label "apply-benchmark-patch" || true diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 2e9980532b..12df3f8ab6 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -1,4 +1,4 @@ -# .github/workflows/benchmarks.yml +# .github/workflows/run-benchmarks.yml name: Validate-Benchmarks on: @@ -20,10 +20,8 @@ jobs: env: SKIP_BENCHMARKS: "0" - AUTO_COMMIT_WEIGHTS: "1" steps: - # ────────────────────────────────────────────────────────────────── - name: Check out PR branch if: ${{ env.SKIP_BENCHMARKS != '1' }} uses: actions/checkout@v4 @@ -124,16 +122,45 @@ jobs: echo "SKIP_BENCHMARKS=1" >> "$GITHUB_ENV" fi + - name: Ensure artifact folder exists + if: ${{ env.SKIP_BENCHMARKS != '1' }} + run: mkdir -p .bench_patch + - name: Run & validate benchmarks if: ${{ env.SKIP_BENCHMARKS != '1' }} - uses: nick-fields/retry@v3 + timeout-minutes: 180 + run: | + chmod +x scripts/benchmark_action.sh + scripts/benchmark_action.sh + + - name: List artifact contents (for debugging) + if: ${{ always() }} + run: | + echo "Workspace: $GITHUB_WORKSPACE" + if [ -d ".bench_patch" ]; then + echo "== .bench_patch ==" + ls -la .bench_patch || true + else + echo ".bench_patch directory is missing" + fi + + - name: Archive bench patch + if: ${{ always() }} + run: | + if [ -d ".bench_patch" ]; then + tar -czf bench-patch.tgz .bench_patch + ls -lh bench-patch.tgz + else + echo "No .bench_patch directory to archive." + fi + + - name: Upload patch artifact (if prepared) + if: ${{ always() }} + uses: actions/upload-artifact@v4 with: - timeout_minutes: 180 - max_attempts: 3 - retry_wait_seconds: 60 - command: | - chmod +x scripts/benchmark_action.sh - scripts/benchmark_action.sh + name: bench-patch + path: bench-patch.tgz + if-no-files-found: warn # (6) — final check after run - name: Check skip label after run diff --git a/pallets/admin-utils/src/benchmarking.rs b/pallets/admin-utils/src/benchmarking.rs index 4d2ac21045..c824a879a5 100644 --- a/pallets/admin-utils/src/benchmarking.rs +++ b/pallets/admin-utils/src/benchmarking.rs @@ -346,5 +346,16 @@ mod benchmarks { _(RawOrigin::Root, 5u16/*version*/)/*sudo_set_commit_reveal_version()*/; } + #[benchmark] + fn sudo_set_owner_immune_neuron_limit() { + pallet_subtensor::Pallet::::init_new_network( + 1u16.into(), /*netuid*/ + 1u16, /*sudo_tempo*/ + ); + + #[extrinsic_call] + _(RawOrigin::Root, 1u16.into()/*netuid*/, 5u16/*immune_neurons*/)/*sudo_set_owner_immune_neuron_limit()*/; + } + //impl_benchmark_test_suite!(AdminUtils, crate::mock::new_test_ext(), crate::mock::Test); } diff --git a/pallets/admin-utils/src/lib.rs b/pallets/admin-utils/src/lib.rs index 1b0407f312..075b8e1976 100644 --- a/pallets/admin-utils/src/lib.rs +++ b/pallets/admin-utils/src/lib.rs @@ -690,7 +690,7 @@ pub mod pallet { /// It is only callable by root and subnet owner. /// The extrinsic will call the Subtensor pallet to set the maximum burn. #[pallet::call_index(23)] - #[pallet::weight(Weight::from_parts(15_940_000, 0) + #[pallet::weight(Weight::from_parts(19_420_000, 0) .saturating_add(::DbWeight::get().reads(2_u64)) .saturating_add(::DbWeight::get().writes(1_u64)))] pub fn sudo_set_max_burn( @@ -1597,8 +1597,8 @@ pub mod pallet { /// Sets the number of immune owner neurons #[pallet::call_index(72)] - #[pallet::weight(Weight::from_parts(15_000_000, 0) - .saturating_add(::DbWeight::get().reads(1_u64)) + #[pallet::weight(Weight::from_parts(4_639_000, 0) + .saturating_add(::DbWeight::get().reads(0_u64)) .saturating_add(::DbWeight::get().writes(1_u64)))] pub fn sudo_set_owner_immune_neuron_limit( origin: OriginFor, diff --git a/pallets/subtensor/src/macros/dispatches.rs b/pallets/subtensor/src/macros/dispatches.rs index c842709208..29654869f1 100644 --- a/pallets/subtensor/src/macros/dispatches.rs +++ b/pallets/subtensor/src/macros/dispatches.rs @@ -665,7 +665,7 @@ mod dispatches { /// - Attempting to set prometheus information withing the rate limit min. /// #[pallet::call_index(40)] - #[pallet::weight((Weight::from_parts(32_310_000, 0) + #[pallet::weight((Weight::from_parts(41_240_000, 0) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)), DispatchClass::Normal, Pays::No))] pub fn serve_axon_tls( @@ -2062,9 +2062,9 @@ mod dispatches { /// * `hotkey` (T::AccountId): /// - The hotkey account to designate as the autostake destination. #[pallet::call_index(114)] - #[pallet::weight( - Weight::from_parts(5_170_000, 0).saturating_add(T::DbWeight::get().writes(1_u64)) - )] + #[pallet::weight((Weight::from_parts(5_170_000, 0) + .saturating_add(T::DbWeight::get().reads(0_u64)) + .saturating_add(T::DbWeight::get().writes(1)), DispatchClass::Normal, Pays::No))] pub fn set_coldkey_auto_stake_hotkey( origin: T::RuntimeOrigin, hotkey: T::AccountId, diff --git a/scripts/benchmark_action.sh b/scripts/benchmark_action.sh index 50b2a3a014..105cbe9bf5 100755 --- a/scripts/benchmark_action.sh +++ b/scripts/benchmark_action.sh @@ -13,10 +13,13 @@ declare -A DISPATCH_PATHS=( THRESHOLD=20 MAX_RETRIES=3 -AUTO_COMMIT="${AUTO_COMMIT_WEIGHTS:-0}" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RUNTIME_WASM="$SCRIPT_DIR/../target/production/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm" +PATCH_DIR="$SCRIPT_DIR/../.bench_patch" +mkdir -p "$PATCH_DIR" + die() { echo "❌ $1" >&2; exit 1; } digits_only() { echo "${1//[^0-9]/}"; } dec() { local d; d=$(digits_only "$1"); echo "$((10#${d:-0}))"; } @@ -32,7 +35,8 @@ patch_weight() { FN="$1" NEWV="$2" perl -0777 -i -pe ' my $n=$ENV{NEWV}; my $hit=0; $hit+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?Weight::from_parts\(\s*)[0-9A-Za-z_]+|$1$n|s; - $hit+=s|(\#\s*\[pallet::weight[^\]]*?Weight::from_parts\(\s*)[0-9A-Za-z_]+(?=[^\]]*?\]\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1$n|s; + # attribute replacement allowing intermediate attributes (e.g., call_index) before pub fn + $hit+=s|(\#\s*\[pallet::weight[^\]]*?Weight::from_parts\(\s*)[0-9A-Za-z_]+(?=[^\]]*\](?:\s*#\[[^\]]*\])*\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1$n|s; END{exit $hit?0:1} ' "$3" || log_warn "patch_weight: no substitution for $1" after=$(sha1sum "$3" | cut -d' ' -f1); [[ "$before" != "$after" ]] @@ -51,36 +55,56 @@ patch_reads_writes() { my $W=$neww eq "" ? $w : u64($neww); return "$pre$R$mid$W$post"; }; - $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?reads_writes\(\s*)([^,]+)(\s*,\s*)([^)]+)|&$rw_sub($1,$2,$3,$4,"")|e; - $h+=s|(\#\s*\[pallet::weight[^\]]*?reads_writes\(\s*)([^,]+)(\s*,\s*)([^)]+)(?=[^\]]*?\]\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|&$rw_sub($1,$2,$3,$4,"")|e; - $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?\.reads\(\s*)([^)]+)|$1.($newr eq "" ? $2 : u64($newr))|e; - $h+=s|(\#\s*\[pallet::weight[^\]]*?\.reads\(\s*)([^)]+)(?=[^\]]*?\]\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1.($newr eq "" ? $2 : u64($newr))|e; - $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?\.writes\(\s*)([^)]+)|$1.($neww eq "" ? $2 : u64($neww))|e; - $h+=s|(\#\s*\[pallet::weight[^\]]*?\.writes\(\s*)([^)]+)(?=[^\]]*?\]\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1.($neww eq "" ? $2 : u64($neww))|e; + # In-body: reads_writes(...) + $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?reads_writes\(\s*)([^,]+)(\s*,\s*)([^)]+)|&$rw_sub($1,$2,$3,$4,"")|sge; + # In-body: .reads(...) and .writes(...) + $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?\.reads\(\s*)([^)]+)|$1.($newr eq "" ? $2 : u64($newr))|sge; + $h+=s|(pub\s+fn\s+\Q$ENV{FN}\E\s*[^{}]*?\.writes\(\s*)([^)]+)|$1.($neww eq "" ? $2 : u64($neww))|sge; + + # Attribute: reads_writes(...), tolerate other attributes between ] and pub fn + $h+=s|(\#\s*\[pallet::weight[^\]]*?reads_writes\(\s*)([^,]+)(\s*,\s*)([^)]+)(?=[^\]]*\](?:\s*#\[[^\]]*\])*\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|&$rw_sub($1,$2,$3,$4,"")|sge; + # Attribute: .reads(...) and .writes(...), tolerate other attributes between ] and pub fn + $h+=s|(\#\s*\[pallet::weight[^\]]*?\.reads\(\s*)([^)]+)(?=[^\]]*\](?:\s*#\[[^\]]*\])*\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1.($newr eq "" ? $2 : u64($newr))|sge; + $h+=s|(\#\s*\[pallet::weight[^\]]*?\.writes\(\s*)([^)]+)(?=[^\]]*\](?:\s*#\[[^\]]*\])*\s*pub\s+fn\s+\Q$ENV{FN}\E\b)|$1.($neww eq "" ? $2 : u64($neww))|sge; + END{exit $h?0:1} ' "$4" || log_warn "patch_reads_writes: no substitution for $1" after=$(sha1sum "$4" | cut -d' ' -f1); [[ "$before" != "$after" ]] } -git_commit_and_push() { - local msg="$1" - local branch; branch=$(git symbolic-ref --quiet --short HEAD || true) - [[ -z "$branch" ]] && die "Not on a branch - cannot push" - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" +write_patch_artifacts_and_fail() { git add "${PATCHED_FILES[@]}" || true - if git diff --cached --quiet; then - echo "ℹ️ No staged changes after patching."; git status --short; return - fi - - echo "==== diff preview ===="; git diff --cached --stat - git diff --cached | head -n 40 || true - echo "======================" + { + echo "Head SHA: $(git rev-parse HEAD)" + echo + echo "==== Benchmark summary ====" + printf '%s\n' "${GLOBAL_SUMMARY[@]}" || true + echo + echo "==== Diffstat ====" + git diff --cached --stat || true + } > "$PATCH_DIR/summary.txt" + + git diff --cached --binary > "$PATCH_DIR/benchmark_patch.diff" || true + + echo "📦 Prepared patch at: $PATCH_DIR/benchmark_patch.diff" + echo "ℹ️ Add the 'apply-benchmark-patch' label to this PR to have CI apply & commit it." + exit 2 +} - git commit -m "$msg" - git push origin "HEAD:$branch" || die "Push to '${branch}' failed." +write_summary_only_and_fail() { + { + echo "Head SHA: $(git rev-parse HEAD)" + echo + echo "==== Benchmark summary ====" + printf '%s\n' "${GLOBAL_SUMMARY[@]}" || true + echo + echo "No auto-patch was generated for the mismatched extrinsics." + echo "Manual update may be required." + } > "$PATCH_DIR/summary.txt" + + echo "⚠️ No patch could be auto-generated. See .bench_patch/summary.txt." + exit 2 } echo "Building runtime-benchmarks…" @@ -91,6 +115,8 @@ echo " Will benchmark pallets: ${PALLET_LIST[*]}" echo "─────────────────────────────────────────────" PATCHED_FILES=() +GLOBAL_SUMMARY=() +ANY_FAILURE=0 ################################################################################ # Benchmark loop @@ -155,13 +181,15 @@ for pallet in "${PALLET_LIST[@]}"; do done < "$TMP"; flush echo; printf ' %s\n' "${summary[@]}" + GLOBAL_SUMMARY+=("${summary[@]}") + (( fail == 0 )) && { echo "✅ '$pallet' within tolerance."; break; } printf ' ❌ %s\n' "${failures[@]}" (( attempt < MAX_RETRIES )) && { echo "→ Retrying …"; (( attempt++ )); continue; } - echo "❌ '$pallet' still failing; patching …" - [[ "$AUTO_COMMIT" != "1" ]] && die "AUTO_COMMIT_WEIGHTS disabled." + echo "❌ '$pallet' still failing; patching (prepare-only) …" + ANY_FAILURE=1 changed=0 for fn in $(printf "%s\n" "${!new_weight[@]}" "${!new_reads[@]}" "${!new_writes[@]}" | sort -u); do @@ -179,11 +207,14 @@ for pallet in "${PALLET_LIST[@]}"; do done ################################################################################ -# Commit & push patches +# Fail if any mismatch; upload artifacts in workflow step ################################################################################ -if (( ${#PATCHED_FILES[@]} )); then - echo -e "\n📦 Committing patched files …" - git_commit_and_push "auto-update benchmark weights" +if (( ANY_FAILURE )); then + if (( ${#PATCHED_FILES[@]} )); then + write_patch_artifacts_and_fail + else + write_summary_only_and_fail + fi fi echo -e "\n══════════════════════════════════════"