ci: add fuzz regression testing and continuous fuzzing infrastructure

.github/workflows/build.yml

@@ -318,6 +318,17 @@ jobs:
       depends-dep-opts: ${{ needs.depends-win64.outputs.dep-opts }}
       runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }}
 
+  test-linux64_fuzz:
+    name: linux64_fuzz-test
+    uses: ./.github/workflows/test-fuzz.yml
+    needs: [check-skip, container, src-linux64_fuzz]
+    if: ${{ vars.SKIP_LINUX64_FUZZ == '' }}
+    with:
+      bundle-key: ${{ needs.src-linux64_fuzz.outputs.key }}
+      build-target: linux64_fuzz
+      container-path: ${{ needs.container.outputs.path }}
+      runs-on: ${{ needs.check-skip.outputs['runner-amd64'] }}
+
   test-linux64:
     name: linux64-test
     uses: ./.github/workflows/test-src.yml

.github/workflows/test-fuzz.yml

@@ -0,0 +1,271 @@
+name: Fuzz regression
+
+on:
+  workflow_call:
+    inputs:
+      bundle-key:
+        description: "Key needed to access bundle of fuzz build artifacts"
+        required: true
+        type: string
+      build-target:
+        description: "Target name as defined by inputs.sh"
+        required: true
+        type: string
+      container-path:
+        description: "Path to built container at registry"
+        required: true
+        type: string
+      runs-on:
+        description: "Runner label to use"
+        required: false
+        default: ubuntu-24.04
+        type: string
+
+jobs:
+  fuzz-regression:
+    name: Fuzz regression
+    runs-on: ${{ inputs.runs-on }}
+    container:
+      image: ${{ inputs.container-path }}
+      options: --user root
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 1
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.bundle-key }}
+
+      - name: Extract build artifacts
+        run: |
+          git config --global --add safe.directory "$PWD"
+          export BUILD_TARGET="${{ inputs.build-target }}"
+          export BUNDLE_KEY="${{ inputs.bundle-key }}"
+          ./ci/dash/bundle-artifacts.sh extract
+        shell: bash
+
+      - name: Download corpus
+        run: |
+          mkdir -p /tmp/fuzz_corpus
+
+          BITCOIN_QA_ASSETS_SHA=ef5d7720f2a1ac20607bd5fba16137f4bfbcfec6
+          DASH_QA_ASSETS_SHA=b4d1a7ce41a9a21d522381944ba84b60b1ad5b60
+
+          # Fail the job if neither external qa-assets source is reachable — otherwise we
+          # silently degrade to synthetic-only/empty-corpus smoke runs and CI goes green on
+          # no signal. Synthetic seeds are additive only; they do not substitute for the
+          # curated external corpora.
+          loaded_external=0
+
+          fetch_pinned_repo() {
+            local repo_url="$1"
+            local repo_sha="$2"
+            local dest_dir="$3"
+            local resolved_sha
+
+            rm -rf "$dest_dir"
+            git init --quiet "$dest_dir"
+            git -C "$dest_dir" remote add origin "$repo_url"
+            git -C "$dest_dir" fetch --depth=1 origin "$repo_sha" || return 1
+            git -C "$dest_dir" checkout --quiet --detach FETCH_HEAD || return 1
+            resolved_sha=$(git -C "$dest_dir" rev-parse HEAD) || return 1
+            echo "Resolved ${repo_url} to ${resolved_sha}"
+            [ "$resolved_sha" = "$repo_sha" ]
+          }
+
+          # Layer 1: bitcoin-core inherited corpus
+          if fetch_pinned_repo https://github.com/bitcoin-core/qa-assets "$BITCOIN_QA_ASSETS_SHA" /tmp/qa-assets; then
+            if [ -d "/tmp/qa-assets/fuzz_seed_corpus" ]; then
+              cp -r /tmp/qa-assets/fuzz_seed_corpus/. /tmp/fuzz_corpus/
+              echo "Loaded bitcoin-core corpus"
+              loaded_external=1
+            fi
+          else
+            echo "::warning::Failed to fetch bitcoin-core/qa-assets at ${BITCOIN_QA_ASSETS_SHA}"
+          fi
+
+          # Layer 2: Dash-specific corpus (overlays on top)
+          if fetch_pinned_repo https://github.com/dashpay/qa-assets "$DASH_QA_ASSETS_SHA" /tmp/dash-qa-assets; then
+            if [ -d "/tmp/dash-qa-assets/fuzz/corpora" ]; then
+              cp -r /tmp/dash-qa-assets/fuzz/corpora/. /tmp/fuzz_corpus/
+              echo "Loaded Dash-specific corpus"
+              loaded_external=1
+            fi
+          else
+            echo "::warning::Failed to fetch dashpay/qa-assets at ${DASH_QA_ASSETS_SHA}"
+          fi
+
+          if [ "$loaded_external" -eq 0 ]; then
+            echo "::error::No external corpus sources reachable - refusing to run synthetic-only/empty-corpus smoke tests that produce no signal"
+            exit 1
+          fi
+
+          # Layer 3: Generate synthetic seeds for Dash-specific targets (additive only —
+          # gated on external corpus being present so we never pass on synthetic-only runs).
+          if [ -f "contrib/fuzz/seed_corpus_from_chain.py" ]; then
+            python3 contrib/fuzz/seed_corpus_from_chain.py --synthetic-only -o /tmp/fuzz_corpus
+          fi
+        shell: bash
+
+      - name: Run fuzz regression tests
+        id: fuzz-test
+        run: |
+          export BUILD_TARGET="${{ inputs.build-target }}"
+          source ./ci/dash/matrix.sh
+
+          BUILD_DIR="build-ci/dashcore-${BUILD_TARGET}"
+          FUZZ_BIN="${BUILD_DIR}/src/test/fuzz/fuzz"
+
+          if [ ! -x "$FUZZ_BIN" ]; then
+            echo "ERROR: Fuzz binary not found at $FUZZ_BIN"
+            exit 1
+          fi
+
+          # Leak detection stays enabled; known-noisy dependency leaks are filtered via
+          # LSAN_OPTIONS suppressions instead of being globally disabled.
+          export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"
+          export LSAN_OPTIONS="suppressions=${BUILD_DIR}/test/sanitizer_suppressions/lsan"
+          export UBSAN_OPTIONS="suppressions=${BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
+
+          # Get list of all targets
+          TARGETS=$(PRINT_ALL_FUZZ_TARGETS_AND_ABORT=1 "$FUZZ_BIN" 2>/tmp/fuzz_target_discovery.err || true)
+          TARGET_COUNT=$(echo "$TARGETS" | grep -c '[^[:space:]]' || true)
+          if [ "$TARGET_COUNT" -eq 0 ]; then
+            if [ -s /tmp/fuzz_target_discovery.err ]; then
+              cat /tmp/fuzz_target_discovery.err
+            fi
+            echo "::error::No fuzz targets found — binary may have failed to start"
+            exit 1
+          fi
+          echo "Found $TARGET_COUNT fuzz targets"
+
+          FAILED=0
+          PASSED=0
+          FAILED_TARGETS=""
+          # libFuzzer writes crash-/leak-/oom-/timeout- files to this directory on failure
+          # via -artifact_prefix, so the "Upload crash artifacts" step below can collect them.
+          ARTIFACT_DIR=/tmp/fuzz_artifacts
+          mkdir -p "$ARTIFACT_DIR"
+
+          # Allowlist of Dash-specific fuzz targets (names beginning with "dash_") that are
+          # permitted to run as smoke-only when no corpus is present. Keep this list small
+          # and documented: every entry is a target we have explicitly decided cannot have
+          # a meaningful corpus yet (e.g. trivially-stateless harnesses). Adding a Dash
+          # target here MUST be justified in review — the default for new dash_* targets
+          # is to fail when the corpus is missing so they do not silently degrade to a
+          # 10-second smoke run that produces no signal.
+          DASH_SMOKE_ONLY_ALLOWLIST=""
+
+          is_dash_smoke_allowed() {
+            local t="$1"
+            for allowed in $DASH_SMOKE_ONLY_ALLOWLIST; do
+              [ "$t" = "$allowed" ] && return 0
+            done
+            return 1
+          }
+
+          while IFS= read -r target; do
+            [ -z "$target" ] && continue
+            corpus_dir="/tmp/fuzz_corpus/${target}"
+            artifact_prefix="${ARTIFACT_DIR}/${target}-"
+
+            # Classify a non-zero exit code from `timeout`/libFuzzer. timeout(1) reports
+            # 124 when the time budget elapsed, and 128+SIGNAL when the child was killed
+            # (137 = SIGKILL, 143 = SIGTERM). Treat those as "timeout/kill" and any other
+            # non-zero status as a generic crash. Both still fail the job.
+            classify_exit() {
+              case "$1" in
+                124|137|143) echo "timeout" ;;
+                *) echo "crash" ;;
+              esac
+            }
+
+            if [ ! -d "$corpus_dir" ] || [ -z "$(ls -A "$corpus_dir" 2>/dev/null)" ]; then
+              # Dash-specific targets (names beginning "dash_") MUST have corpus inputs from
+              # either the pinned dashpay/qa-assets layer or the synthetic seeder. Falling
+              # back to a 10-second empty-corpus smoke run produces no real signal and was
+              # masking missing-corpus regressions for newly added Dash harnesses. Inherited
+              # upstream/non-Dash targets are still allowed the smoke fallback so we do not
+              # regress on bitcoin-core targets that legitimately ship without a corpus.
+              case "$target" in
+                dash_*)
+                  if ! is_dash_smoke_allowed "$target"; then
+                    echo "::error::FAIL: $target has no corpus inputs in /tmp/fuzz_corpus/${target} — Dash-specific targets must ship with corpus data (qa-assets or synthetic seeder); refusing to silently smoke-test"
+                    FAILED=$((FAILED + 1))
+                    FAILED_TARGETS="${FAILED_TARGETS}  - ${target} (missing-corpus)\n"
+                    continue
+                  fi
+                  echo "::warning::${target} is on the documented Dash smoke-only allowlist — running 10s empty-corpus check"
+                  ;;
+              esac
+              # No corpus for this target — run with empty input for 10s
+              # This catches basic initialization crashes
+              echo "::group::${target} (empty corpus, 10s run)"
+              mkdir -p "$corpus_dir"
+              # timeout(30) intentionally exceeds -max_total_time=10 to absorb startup/teardown jitter
+              # while still terminating genuinely hung processes.
+              if FUZZ="$target" timeout 30 "$FUZZ_BIN" \
+                  -rss_limit_mb=4000 \
+                  -max_total_time=10 \
+                  -reload=0 \
+                  -artifact_prefix="$artifact_prefix" \
+                  "$corpus_dir" 2>&1; then
+                echo "PASS: $target (empty corpus)"
+                PASSED=$((PASSED + 1))
+              else
+                EXIT_CODE=$?
+                KIND=$(classify_exit "$EXIT_CODE")
+                echo "::error::FAIL: $target exited with code $EXIT_CODE (${KIND})"
+                FAILED=$((FAILED + 1))
+                FAILED_TARGETS="${FAILED_TARGETS}  - ${target} (${KIND}, exit code ${EXIT_CODE})\n"
+              fi
+              echo "::endgroup::"
+              continue
+            fi
+
+            # Run corpus regression (replay all inputs)
+            echo "::group::${target} ($(find "$corpus_dir" -maxdepth 1 -type f | wc -l) inputs)"
+            if FUZZ="$target" timeout 3600 "$FUZZ_BIN" \
+                -rss_limit_mb=4000 \
+                -runs=1 \
+                -artifact_prefix="$artifact_prefix" \
+                "$corpus_dir" 2>&1; then
+              echo "PASS: $target"
+              PASSED=$((PASSED + 1))
+            else
+              EXIT_CODE=$?
+              KIND=$(classify_exit "$EXIT_CODE")
+              echo "::error::FAIL: $target exited with code $EXIT_CODE (${KIND})"
+              FAILED=$((FAILED + 1))
+              FAILED_TARGETS="${FAILED_TARGETS}  - ${target} (${KIND}, exit code ${EXIT_CODE})\n"
+            fi
+            echo "::endgroup::"
+          done <<< "$TARGETS"
+
+          echo ""
+          echo "=== Fuzz Regression Summary ==="
+          echo "Passed: $PASSED"
+          echo "Failed: $FAILED"
+          echo "Total: $TARGET_COUNT"
+
+          if [ $FAILED -gt 0 ]; then
+            echo ""
+            echo "=== Failed Targets ==="
+            printf '%b' "$FAILED_TARGETS"
+            echo "::error::$FAILED fuzz target(s) failed regression testing"
+            exit 1
+          fi
+        shell: bash
+
+      - name: Upload crash artifacts
+        if: failure() && steps.fuzz-test.conclusion == 'failure'
+        uses: actions/upload-artifact@v4
+        with:
+          name: fuzz-crashes-${{ inputs.build-target }}
+          path: /tmp/fuzz_artifacts
+          if-no-files-found: ignore
+          retention-days: 30

contrib/fuzz/README.md

@@ -0,0 +1,103 @@
+# Dash Core Fuzz Testing Tools
+
+This directory contains tools for continuous fuzz testing of Dash Core.
+
+## Overview
+
+Dash Core inherits ~100 fuzz targets from Bitcoin Core and adds Dash-specific
+targets for:
+- Special transaction serialization (ProTx, CoinJoin, Asset Lock/Unlock, etc.)
+- BLS operations and IES encryption
+- LLMQ/DKG message handling
+- Governance object validation
+- Masternode list management
+
+Some Dash-specific fuzz targets are planned/in-progress. Corpus tooling
+pre-generates synthetic seeds for those target names so coverage is ready when
+the targets are added.
+
+## Tools
+
+### `continuous_fuzz_daemon.sh`
+
+A daemon script that continuously cycles through all fuzz targets with persistent
+corpus storage and crash detection.
+
+```bash
+# Run all targets, 10 minutes each, indefinitely
+./continuous_fuzz_daemon.sh --fuzz-bin /path/to/fuzz --time-per-target 600
+
+# Run specific targets only
+./continuous_fuzz_daemon.sh --targets bls_operations,bls_ies --time-per-target 3600
+
+# Single cycle (good for cron)
+./continuous_fuzz_daemon.sh --single-cycle --time-per-target 300
+
+# Dry run — list targets
+./continuous_fuzz_daemon.sh --dry-run
+```
+
+**Output directories:**
+- `~/fuzz_corpus/<target>/` — persistent corpus per target
+- `~/fuzz_crashes/<target>/` — crash artifacts (crash-*, timeout-*, oom-*)
+- `~/fuzz_logs/` — per-target logs and daemon log
+
+### `seed_corpus_from_chain.py`
+
+Extracts real-world data from a running Dash node into fuzzer-consumable corpus
+files. Connects via `dash-cli` RPC.
+
+```bash
+# Extract from a running node
+./seed_corpus_from_chain.py -o /path/to/corpus --blocks 500
+
+# Generate only synthetic seeds (no running node required)
+./seed_corpus_from_chain.py -o /path/to/corpus --synthetic-only
+```
+
+**What it extracts:**
+- Serialized blocks and block headers
+- Special transactions (ProRegTx, ProUpServTx, CoinJoin, Asset Lock, etc.)
+- Governance objects and votes
+- Masternode list entries
+- Quorum commitment data
+
+## CI Integration
+
+The `test-fuzz.yml` workflow runs fuzz regression tests on every PR:
+
+1. Builds fuzz targets with sanitizers (ASan + UBSan + libFuzzer)
+2. Downloads seed corpus from `bitcoin-core/qa-assets` + synthetic Dash seeds
+3. Replays all corpus inputs against every fuzz target
+4. Reports failures as CI errors
+
+This catches regressions in seconds — any code change that causes a previously-
+working input to crash will be caught.
+
+## Building Fuzz Targets
+
+```bash
+# Configure with fuzzing + sanitizers
+./configure --enable-fuzz --with-sanitizers=fuzzer,address,undefined \
+    CC='clang -ftrivial-auto-var-init=pattern' \
+    CXX='clang++ -ftrivial-auto-var-init=pattern'
+
+# Build
+make -j$(nproc)
+
+# The fuzz binary is at src/test/fuzz/fuzz
+# Select target with FUZZ=<target_name>
+FUZZ=bls_operations ./src/test/fuzz/fuzz corpus_dir/
+```
+
+## Contributing Corpus Inputs
+
+Found an interesting input? Add it to the appropriate corpus directory:
+
+```bash
+# The filename should be the sha256 of the content (for dedup)
+sha256sum input_file
+cp input_file fuzz_corpus/<target_name>/<sha256_prefix>
+```
+
+Crash-reproducing inputs are especially valuable — they become regression tests.