From c4a17c62c2511f47ebe0d36d1f0f14d0b03d8da1 Mon Sep 17 00:00:00 2001 From: Nicholas Hulston Date: Mon, 19 May 2025 13:13:26 -0400 Subject: [PATCH 1/3] add fallback for lambda --- .../debugger/exception/DefaultExceptionDebugger.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java index 2d4b2dd5fe5..9dcef9b90e5 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java @@ -148,6 +148,13 @@ private static void processSnapshotsAndSetTags( int[] mapping = createThrowableMapping(currentEx, t); StackTraceElement[] innerTrace = currentEx.getStackTrace(); int currentIdx = innerTrace.length - snapshot.getStack().size(); + + if (currentIdx < 0) { + // This means the innerTrace was truncated by the underlying environment. + // This is known to happen in AWS Lambda, but may also happen elsewhere. + currentIdx = i; + } + if (!sanityCheckSnapshotAssignment(snapshot, innerTrace, currentIdx)) { continue; } From a60af39e5967d8bd23c3be8ce4e1a0e0ee4a8589 Mon Sep 17 00:00:00 2001 From: Nicholas Hulston Date: Mon, 19 May 2025 13:14:09 -0400 Subject: [PATCH 2/3] unit test --- .../DefaultExceptionDebuggerTest.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java index aaabb300cc0..871703c89a5 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java @@ -280,6 +280,39 @@ public void filteringOutErrors() { verify(manager, times(0)).isAlreadyInstrumented(any()); } + @Test + public void lambdaTruncatedInnerTraceFallback() { + RuntimeException exception = new RuntimeException("lambda"); + String fingerprint = Fingerprinter.fingerprint(exception, classNameFiltering); + AgentSpan span = mock(AgentSpan.class); + doAnswer(this::recordTags).when(span).setTag(anyString(), anyString()); + when(span.getTag(anyString())).thenAnswer(inv -> spanTags.get(inv.getArgument(0))); + when(span.getTags()).thenReturn(spanTags); + + exceptionDebugger.handleException(exception, span); + assertWithTimeout( + () -> exceptionDebugger.getExceptionProbeManager().isAlreadyInstrumented(fingerprint), + Duration.ofSeconds(30)); + + generateSnapshots(exception); + + ExceptionProbeManager.ThrowableState state = + exceptionDebugger.getExceptionProbeManager().getStateByThrowable(exception); + List snapshots = state.getSnapshots(); + StackTraceElement ste = exception.getStackTrace()[0]; + CapturedStackFrame dummyFrame = CapturedStackFrame.from(ste); + for (Snapshot snap : snapshots) { + snap.getStack().add(0, dummyFrame); + } + + // This should hit the `currentIdx < 0` branch and fallback to i=0 + exceptionDebugger.handleException(exception, span); + + String tagName = String.format(SNAPSHOT_ID_TAG_FMT, 0); + assertTrue(spanTags.containsKey(tagName)); + assertEquals(snapshots.get(0).getId(), spanTags.get(tagName)); + } + private Object recordTags(InvocationOnMock invocationOnMock) { Object[] args = invocationOnMock.getArguments(); String key = (String) args[0]; From 03689f9b151ff3b369aab18f98f519b9c06e0a07 Mon Sep 17 00:00:00 2001 From: Nicholas Hulston Date: Tue, 20 May 2025 10:45:53 -0400 Subject: [PATCH 3/3] mock exception to return truncated stack like Lambda --- .../DefaultExceptionDebuggerTest.java | 80 ++++++++++++++----- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java index 871703c89a5..f32bfc75475 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/exception/DefaultExceptionDebuggerTest.java @@ -30,6 +30,8 @@ import datadog.trace.bootstrap.debugger.CapturedContext; import datadog.trace.bootstrap.debugger.CapturedStackFrame; import datadog.trace.bootstrap.debugger.MethodLocation; +import datadog.trace.bootstrap.debugger.ProbeImplementation; +import datadog.trace.bootstrap.debugger.ProbeLocation; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import java.io.BufferedReader; import java.io.IOException; @@ -282,35 +284,73 @@ public void filteringOutErrors() { @Test public void lambdaTruncatedInnerTraceFallback() { - RuntimeException exception = new RuntimeException("lambda"); - String fingerprint = Fingerprinter.fingerprint(exception, classNameFiltering); AgentSpan span = mock(AgentSpan.class); doAnswer(this::recordTags).when(span).setTag(anyString(), anyString()); when(span.getTag(anyString())).thenAnswer(inv -> spanTags.get(inv.getArgument(0))); when(span.getTags()).thenReturn(spanTags); - exceptionDebugger.handleException(exception, span); - assertWithTimeout( - () -> exceptionDebugger.getExceptionProbeManager().isAlreadyInstrumented(fingerprint), - Duration.ofSeconds(30)); - - generateSnapshots(exception); - - ExceptionProbeManager.ThrowableState state = - exceptionDebugger.getExceptionProbeManager().getStateByThrowable(exception); - List snapshots = state.getSnapshots(); - StackTraceElement ste = exception.getStackTrace()[0]; - CapturedStackFrame dummyFrame = CapturedStackFrame.from(ste); - for (Snapshot snap : snapshots) { - snap.getStack().add(0, dummyFrame); + // Create an exception with a real truncated stack trace from Lambda + RuntimeException lambdaException = + new RuntimeException("lambda") { + @Override + public StackTraceElement[] getStackTrace() { + return new StackTraceElement[] { + new StackTraceElement("Main", "handleRequest", "Main.java", 11), + new StackTraceElement( + "jdk.internal.reflect.DirectMethodHandleAccessor", + "invoke", + "Unknown Source", + -1), + new StackTraceElement("java.lang.reflect.Method", "invoke", "Unknown Source", -1) + }; + } + }; + + // Set up the snapshot with a longer stack to represent original data + List snapshotStack = new ArrayList<>(); + snapshotStack.add( + CapturedStackFrame.from(new StackTraceElement("Main", "handleRequest", "Main.java", 11))); + for (int i = 0; i < 5; i++) { + snapshotStack.add( + CapturedStackFrame.from( + new StackTraceElement("Lambda.Frame" + i, "method", "Lambda.java", 100 + i))); } - // This should hit the `currentIdx < 0` branch and fallback to i=0 - exceptionDebugger.handleException(exception, span); - + // Mock snapshot + Snapshot mockSnapshot = mock(Snapshot.class); + when(mockSnapshot.getId()).thenReturn("test-snapshot-id"); + when(mockSnapshot.getStack()).thenReturn(snapshotStack); + when(mockSnapshot.getChainedExceptionIdx()).thenReturn(0); + + // Mock probe + ProbeLocation mockLocation = mock(ProbeLocation.class); + when(mockLocation.getType()).thenReturn("Main"); + when(mockLocation.getMethod()).thenReturn("handleRequest"); + ProbeImplementation mockProbe = mock(ProbeImplementation.class); + when(mockProbe.getLocation()).thenReturn(mockLocation); + when(mockSnapshot.getProbe()).thenReturn(mockProbe); + + // Mock exception state + ExceptionProbeManager.ThrowableState state = mock(ExceptionProbeManager.ThrowableState.class); + when(state.getSnapshots()).thenReturn(singletonList(mockSnapshot)); + when(state.getExceptionId()).thenReturn("test-exception-id"); + when(state.isSnapshotSent()).thenReturn(false); + + // Create mock manager that returns our state + ExceptionProbeManager mockManager = mock(ExceptionProbeManager.class); + when(mockManager.isAlreadyInstrumented(anyString())).thenReturn(true); + when(mockManager.getStateByThrowable(lambdaException)).thenReturn(state); + + DefaultExceptionDebugger testDebugger = + new DefaultExceptionDebugger(mockManager, configurationUpdater, classNameFiltering, 100); + + // Test + testDebugger.handleException(lambdaException, span); + + // Verify String tagName = String.format(SNAPSHOT_ID_TAG_FMT, 0); assertTrue(spanTags.containsKey(tagName)); - assertEquals(snapshots.get(0).getId(), spanTags.get(tagName)); + assertEquals("test-snapshot-id", spanTags.get(tagName)); } private Object recordTags(InvocationOnMock invocationOnMock) {