Add comprehensive unwinding validation with advanced reporting

.github/scripts/prepare_reports.sh

@@ -1,11 +1,15 @@
   #!/usr/bin/env bash
 
 set -e
-mkdir -p reports
-cp /tmp/hs_err* reports/ || true
-cp ddprof-test/javacore*.txt reports/ || true
-cp ddprof-test/build/hs_err* reports/ || true
-cp -r ddprof-lib/build/tmp reports/native_build || true
-cp -r ddprof-test/build/reports/tests reports/tests || true
-cp -r /tmp/recordings reports/recordings || true
-find ddprof-lib/build -name 'libjavaProfiler.*' -exec cp {} reports/ \; || true
+mkdir -p test-reports
+mkdir -p unwinding-reports
+cp /tmp/hs_err* test-reports/ || true
+cp ddprof-test/javacore*.txt test-reports/ || true
+cp ddprof-test/build/hs_err* test-reports/ || true
+cp -r ddprof-lib/build/tmp test-reports/native_build || true
+cp -r ddprof-test/build/reports/tests test-reports/tests || true
+cp -r /tmp/recordings test-reports/recordings || true
+find ddprof-lib/build -name 'libjavaProfiler.*' -exec cp {} test-reports/ \; || true
+
+cp -r ddprof-test/build/reports/unwinding-summary.md unwinding-reports/ || true
+cp -r /tmp/unwinding-recordings/* unwinding-reports/ || true

.github/scripts/unwinding_report_alpine_aarch64.sh

@@ -0,0 +1,34 @@
+#! /bin/sh
+
+set -e
+set +x
+
+export KEEP_JFRS=true
+export TEST_COMMIT="${1}"
+export TEST_CONFIGURATION="${2}"
+export LIBRARY="musl"
+export CONFIG="${3}"
+export JAVA_HOME="${4}"
+export JAVA_TEST_HOME="${5}"
+
+export PATH="${JAVA_HOME}/bin":${PATH}
+
+# due to env hell in GHA containers, we need to re-do the logic from Extract Versions here
+JAVA_VERSION=$("${JAVA_TEST_HOME}/bin/java" -version 2>&1 | awk -F '"' '/version/ {
+  split($2, v, "[._]");
+  if (v[2] == "") {
+    # Version is like "24": assume it is major only and add .0.0
+    printf "%s.0.0\n", v[1]
+  } else if (v[1] == "1") {
+    # Java 8 or older: Format is "1.major.minor_update"
+    printf "%s.%s.%s\n", v[2], v[3], v[4]
+  } else {
+    # Java 9 or newer: Format is "major.minor.patch"
+    printf "%s.%s.%s\n", v[1], v[2], v[3]
+  }
+}')
+export JAVA_VERSION
+
+apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock tar binutils >/dev/null
+
+./gradlew -PCI :ddprof-test:unwindingReport --no-daemon --parallel --build-cache --no-watch-fs

.github/workflows/test_workflow.yml

@@ -100,25 +100,43 @@ jobs:
             echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64" >> failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt
             exit 1
           fi
-      - uses: actions/upload-artifact@v4
+      - name: Generate Unwinding Report
+        if: success() && matrix.config == 'debug'
+        run: |
+          ./gradlew -PCI :ddprof-test:unwindingReport --no-daemon
+      - name: Add Unwinding Report to Job Summary
+        if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != ''
+        run: |
+          echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (amd64)" >> $GITHUB_STEP_SUMMARY
+          cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
         if: success()
         with:
           name: (build) test-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }})
           path: build/
-      - uses: actions/upload-artifact@v4
+      - name: Upload failures
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: failures-glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64
           path: failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt
       - name: Prepare reports
-        if: failure()
+        if: always()
         run: |
           .github/scripts/prepare_reports.sh
-      - uses: actions/upload-artifact@v4
+      - name: Upload unwinding reports
+        uses: actions/upload-artifact@v4
+        if: success() && matrix.config == 'debug'
+        with:
+          name: (unwinding-reports) unwinding-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }})
+          path: unwinding-reports
+      - name: Upload test reports
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: (reports) test-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }})
-          path: reports
+          path: test-reports
 
   test-linux-musl-amd64:
     needs: cache-jdks
@@ -203,25 +221,43 @@ jobs:
             echo "musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64" >> failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt
             exit 1
           fi
-      - uses: actions/upload-artifact@v4
+      - name: Generate Unwinding Report
+        if: success() && matrix.config == 'debug'
+        run: |
+          ./gradlew -PCI :ddprof-test:unwindingReport --no-daemon
+      - name: Add Unwinding Report to Job Summary
+        if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != ''
+        run: |
+          echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (amd64-musl)" >> $GITHUB_STEP_SUMMARY
+          cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
         if: success()
         with:
           name: (build) test-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }})
           path: build/
-      - uses: actions/upload-artifact@v4
+      - name: Upload failures
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: failures-musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64
           path: failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt
       - name: Prepare reports
-        if: failure()
+        if: always()
         run: |
           .github/scripts/prepare_reports.sh
-      - uses: actions/upload-artifact@v4
+      - name: Upload unwinding reports
+        uses: actions/upload-artifact@v4
+        if: success() && matrix.config == 'debug'
+        with:
+          name: (unwinding-reports) unwinding-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }})
+          path: unwinding-reports
+      - name: Upload test reports
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: (reports) test-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }})
-          path: reports
+          name: (test-reports) test-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }})
+          path: test-reports
 
   test-linux-glibc-aarch64:
     needs: cache-jdks
@@ -314,25 +350,43 @@ jobs:
             echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64" >> failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt
             exit 1
           fi
-      - uses: actions/upload-artifact@v4
+      - name: Generate Unwinding Report
+        if: success() && matrix.config == 'debug'
+        run: |
+          ./gradlew -PCI :ddprof-test:unwindingReport --no-daemon
+      - name: Add Unwinding Report to Job Summary
+        if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != ''
+        run: |
+          echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (aarch64)" >> $GITHUB_STEP_SUMMARY
+          cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
         if: success()
         with:
           name: (build) test-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }})
           path: build/
-      - uses: actions/upload-artifact@v4
+      - name: Upload failures
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: failures-glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64
           path: failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt
       - name: Prepare reports
-        if: failure()
+        if: always()
         run: |
           .github/scripts/prepare_reports.sh
-      - uses: actions/upload-artifact@v4
+      - name: Upload unwinding reports
+        uses: actions/upload-artifact@v4
+        if: success() && matrix.config == 'debug'
+        with:
+          name: (unwinding-reports) unwinding-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }})
+          path: unwinding-reports
+      - name: Upload test reports
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: (reports) test-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }})
-          path: reports
+          name: (test-reports) test-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }})
+          path: test-reports
 
   test-linux-musl-aarch64:
      needs: cache-jdks
@@ -394,22 +448,44 @@ jobs:
              echo "musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64" >> failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt
              exit 1
            fi
-       - uses: actions/upload-artifact@v4
+       - name: Generate Unwinding Report
+         if: success() && matrix.config == 'debug'
+         run: |
+           docker run --cpus 4 --rm -v /tmp:/tmp -v "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" -w "${GITHUB_WORKSPACE}" alpine:3.21 /bin/sh -c "
+             \"$GITHUB_WORKSPACE/.github/scripts/unwinding_report_alpine_aarch64.sh\" \
+             \"${{ github.sha }}\" \"musl/${{ matrix.java_version }}-${{ matrix.config }}-aarch64\" \
+             \"${{ matrix.config }}\" \"${{ env.JAVA_HOME }}\" \"${{ env.JAVA_TEST_HOME }}\"
+           "
+       - name: Add Unwinding Report to Job Summary
+         if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != ''
+         run: |
+           echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (aarch64-musl)" >> $GITHUB_STEP_SUMMARY
+           cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY
+       - name: Upload build artifacts
+         uses: actions/upload-artifact@v4
          if: success()
          with:
            name: (build) test-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }})
            path: build/
-       - uses: actions/upload-artifact@v4
+       - name: Upload failures
+         uses: actions/upload-artifact@v4
          if: failure()
          with:
            name: failures-musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64
            path: failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt
        - name: Prepare reports
-         if: failure()
+         if: always()
          run: |
            .github/scripts/prepare_reports.sh
-       - uses: actions/upload-artifact@v4
+       - name: Upload unwinding reports
+         uses: actions/upload-artifact@v4
+         if: success() && matrix.config == 'debug'
+         with:
+           name: (unwinding-reports) unwinding-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }})
+           path: unwinding-reports
+       - name: Upload test reports
+         uses: actions/upload-artifact@v4
          if: failure()
          with:
-           name: (reports) test-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }})
-           path: reports
+           name: (test-reports) test-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }})
+           path: test-reports

CLAUDE.md

@@ -306,3 +306,6 @@ With separate debug symbol packages for production debugging support.
 - Always challange my proposals. Use deep analysis and logic to find flaws in what I am proposing
 
 - Exclude ddprof-lib/build/async-profiler from searches of active usage
+
+- Run tests with 'testdebug' gradle task
+- Use at most Java 21 to build and run tests

README.md

@@ -86,6 +86,85 @@ The project includes both Java and C++ unit tests. You can run them using:
 ### Cross-JDK Testing
 `JAVA_TEST_HOME=<path to test JDK> ./gradlew testDebug`
 
+## Unwinding Validation Tool
+
+The project includes a comprehensive unwinding validation tool that tests JIT compilation unwinding scenarios to detect stack frame issues. This tool validates the profiler's ability to correctly unwind stack frames during complex JIT compilation scenarios.
+
+### Running the Unwinding Validator
+
+```bash
+# Run all unwinding validation scenarios (release or debug build required)
+./gradlew :ddprof-test:runUnwindingValidator
+
+# Run specific scenario
+./gradlew :ddprof-test:runUnwindingValidator -PvalidatorArgs="--scenario=C2CompilationTriggers"
+
+# Generate markdown report for CI
+./gradlew :ddprof-test:unwindingReport
+
+# Show available options
+./gradlew :ddprof-test:runUnwindingValidator -PvalidatorArgs="--help"
+```
+
+### Available Scenarios
+
+The validator includes 13 specialized scenarios targeting different unwinding challenges:
+
+- **C2CompilationTriggers** - Heavy computational workloads that trigger C2 compilation
+- **OSRScenarios** - On-Stack Replacement compilation scenarios
+- **ConcurrentC2Compilation** - Concurrent C2 compilation stress testing
+- **C2DeoptScenarios** - C2 deoptimization and transition edge cases
+- **ExtendedJNIScenarios** - Extended JNI operation patterns
+- **MultipleStressRounds** - Multiple concurrent stress rounds
+- **ExtendedPLTScenarios** - PLT (Procedure Linkage Table) resolution scenarios
+- **ActivePLTResolution** - Intensive PLT resolution during profiling
+- **ConcurrentCompilationStress** - Heavy JIT compilation + native activity
+- **VeneerHeavyScenarios** - ARM64 veneer/trampoline intensive workloads
+- **RapidTierTransitions** - Rapid compilation tier transitions
+- **DynamicLibraryOps** - Dynamic library operations during profiling
+- **StackBoundaryStress** - Stack boundary stress scenarios
+
+### Output Formats
+
+The validator supports multiple output formats:
+
+```bash
+# Text output (default)
+./gradlew :ddprof-test:runUnwindingValidator
+
+# JSON format for programmatic analysis
+./gradlew :ddprof-test:runUnwindingValidator -PvalidatorArgs="--output-format=json --output-file=unwinding-report.json"
+
+# Markdown format for documentation
+./gradlew :ddprof-test:runUnwindingValidator -PvalidatorArgs="--output-format=markdown --output-file=unwinding-report.md"
+```
+
+### CI Integration
+
+The unwinding validator is automatically integrated into GitHub Actions CI pipeline:
+
+- Runs only on **debug builds** in CI (provides clean measurements without optimization interference)
+- Generates rich markdown reports displayed directly in job summaries
+- Creates downloadable report artifacts for historical analysis
+- Fails builds when critical unwinding issues are detected
+
+The validator provides immediate visibility into unwinding quality across all supported platforms and Java versions without requiring artifact downloads.
+
+### Understanding Results
+
+The tool analyzes JFR (Java Flight Recorder) data to measure:
+
+- **Error Rate** - Percentage of samples with unwinding failures (`.unknown()`, `.break_interpreted()`)
+- **Native Coverage** - Percentage of samples successfully unwound in native code
+- **Sample Count** - Total profiling samples captured during validation
+- **Error Types** - Breakdown of specific unwinding failure patterns
+
+Results are categorized as:
+- 🟢 **Excellent** - Error rate < 0.1%
+- 🟢 **Good** - Error rate < 1.0%
+- 🟡 **Moderate** - Error rate < 5.0%
+- 🔴 **Needs Work** - Error rate ≥ 5.0%
+
 ## Release Builds and Debug Information
 
 ### Split Debug Information

build.gradle

@@ -59,7 +59,7 @@ subprojects {
 apply from: rootProject.file('common.gradle')
 apply from: rootProject.file('gradle/configurations.gradle')
 
-def isCI = System.getenv("CI") != null
+def isCI = project.hasProperty("CI") || Boolean.parseBoolean(System.getenv("CI"))
 
 nexusPublishing {
   repositories {