test: e2e replay test using Maestro (js)

dev-packages/e2e-tests/maestro/captureException.yml

@@ -1,4 +1,5 @@
 appId: ${APP_ID}
+jsEngine: graaljs
 ---
 - runFlow: utils/launchTestAppClear.yml
 - tapOn: "Capture Exception"

dev-packages/e2e-tests/maestro/captureMessage.yml

@@ -1,4 +1,5 @@
 appId: ${APP_ID}
+jsEngine: graaljs
 ---
 - runFlow: utils/launchTestAppClear.yml
 - tapOn: "Capture Message"

dev-packages/e2e-tests/maestro/captureReplay.yml

@@ -0,0 +1,13 @@
+appId: ${APP_ID}
+jsEngine: graaljs
+---
+- runFlow:
+    file: utils/launchTestAppClear.yml
+    env:
+      replaysOnErrorSampleRate: 1.0
+- tapOn: "Capture Exception"
+- runFlow: utils/assertEventIdVisible.yml
+- runFlow:
+    file: utils/assertReplay.yml
+    when:
+      platform: iOS

dev-packages/e2e-tests/maestro/captureUnhandledPromiseRejection.yml

@@ -1,4 +1,5 @@
 appId: ${APP_ID}
+jsEngine: graaljs
 ---
 - runFlow: utils/launchTestAppClear.yml
 - tapOn: "Unhandled Promise Rejection"

dev-packages/e2e-tests/maestro/close.yml

@@ -1,4 +1,5 @@
 appId: ${APP_ID}
+jsEngine: graaljs
 ---
 - runFlow: utils/launchTestAppClear.yml
 - tapOn: "Close"

dev-packages/e2e-tests/maestro/crash.yml

@@ -1,4 +1,5 @@
 appId: ${APP_ID}
+jsEngine: graaljs
 ---
 - runFlow: utils/launchTestAppClear.yml
 - tapOn: "Crash"

dev-packages/e2e-tests/maestro/utils/assertEventIdVisible.yml

@@ -1,4 +1,5 @@
 appId: ${APP_ID}
+jsEngine: graaljs
 ---
 - extendedWaitUntil:
     visible:
@@ -8,3 +9,12 @@ appId: ${APP_ID}
 - copyTextFrom:
     id: "eventId"
 - assertTrue: ${maestro.copiedText}
+
+- runScript:
+    file: sentryApi.js
+    env:
+      fetch: event
+      id: ${maestro.copiedText}
+      sentryAuthToken: ${SENTRY_AUTH_TOKEN}
+
+- assertTrue: ${output.eventId == maestro.copiedText}

dev-packages/e2e-tests/maestro/utils/assertReplay.yml

@@ -0,0 +1,23 @@
+appId: ${APP_ID}
+jsEngine: graaljs
+---
+- extendedWaitUntil:
+    visible:
+      id: "eventId"
+    timeout: 60_000 # 60 seconds
+
+- copyTextFrom:
+    id: "eventId"
+- assertTrue: ${maestro.copiedText}
+
+- runScript:
+    file: sentryApi.js
+    env:
+      fetch: replay
+      eventId: ${maestro.copiedText}
+      sentryAuthToken: ${SENTRY_AUTH_TOKEN}
+
+- assertTrue: ${output.replayId}
+- assertTrue: ${output.replayDuration}
+- assertTrue: ${output.replaySegments}
+- assertTrue: ${output.replayCodec == "ftypmp42"}

dev-packages/e2e-tests/maestro/utils/launchTestAppClear.yml

@@ -1,9 +1,15 @@
 appId: ${APP_ID}
+jsEngine: graaljs
 ---
+# Ensure the app is killed, otherwise we may see "INTERNAL: UiAutomation not connected" errors on Android.
+# They seem to be casued by a previous test case running for a long time without UI interactions (e.g. runScript).
+- killApp
+
 - launchApp:
     clearState: true
     arguments:
       sentryAuthToken: ${SENTRY_AUTH_TOKEN}
+      replaysOnErrorSampleRate: ${replaysOnErrorSampleRate}
 
 - extendedWaitUntil:
     visible: "E2E Tests Ready"

dev-packages/e2e-tests/maestro/utils/sentryApi.js

@@ -0,0 +1,80 @@
+const baseUrl = 'https://sentry.io/api/0/projects/sentry-sdks/sentry-react-native';
+
+const RETRY_COUNT = 600;
+const RETRY_INTERVAL = 1000;
+const requestHeaders = { 'Authorization': `Bearer ${sentryAuthToken}` }
+
+function sleep(ms) {
+  // TODO reach out to Maestro & GrallJS via GitHub issues.
+  //   return new Promise(resolve => setTimeout(resolve, ms));
+  // Instead, we need to do a busy wait.
+  const until = Date.now() + ms;
+  while (Date.now() < until) {
+    // console.log(`Sleeping for ${until - Date.now()} ms`);
+    try {
+      http.get('http://127.0.0.1:1');
+    } catch (e) {
+      // Ignore
+    }
+  }
+}
+
+function fetchFromSentry(url) {
+  console.log(`Fetching ${url}`);
+  let retries = 0;
+  const shouldRetry = (response) => {
+    switch (response.status) {
+      case 200:
+        return false;
+      case 403:
+        throw new Error(`Could not fetch ${url}: ${response.status} | ${response.body}`);
+      default:
+        if (retries++ < RETRY_COUNT) {
+          console.log(`Request failed (HTTP ${response.status}), retrying: ${retries}/${RETRY_COUNT}`);
+          return true;
+        }
+        throw new Error(`Could not fetch ${url} within retry limit: ${response.status} | ${response.body}`);
+    }
+  }
+
+  while (true) {
+    const response = http.get(url, { headers: requestHeaders })
+    if (!shouldRetry(response)) {
+      console.log(`Received HTTP ${response.status}: body length ${response.body.length}`);
+      return response.body;
+    }
+    sleep(RETRY_INTERVAL);
+  }
+};
+
+function setOutput(data) {
+  for (const [key, value] of Object.entries(data)) {
+    console.log(`Setting output.${key} = '${value}'`);
+    output[key] = value;
+  }
+}
+
+// Note: "fetch", "id", "eventId", etc. are script inputs, see for example assertEventIdIVisible.yml
+switch (fetch) {
+  case 'event': {
+    const data = json(fetchFromSentry(`${baseUrl}/events/${id}/json/`));
+    setOutput({ eventId: data.event_id });
+    break;
+  }
+  case 'replay': {
+    const event = json(fetchFromSentry(`${baseUrl}/events/${eventId}/json/`));
+    const replayId = event._dsc.replay_id.replace(/\-/g, '');
+    const replay = json(fetchFromSentry(`${baseUrl}/replays/${replayId}/`));
+    const segment = fetchFromSentry(`${baseUrl}/replays/${replayId}/videos/0/`);
+
+    setOutput({
+      replayId: replay.data.id,
+      replayDuration: replay.data.duration,
+      replaySegments: replay.data.count_segments,
+      replayCodec: segment.slice(4, 12)
+    });
+    break;
+  }
+  default:
+    throw new Error(`Unknown "fetch" value: '${fetch}'`);
+}

dev-packages/e2e-tests/patch-scripts/rn.patch.app.js

@@ -21,11 +21,18 @@ logger.info('Patching RN App.(js|tsx)', args.app);
 const initPatch = `
 import * as Sentry from '@sentry/react-native';
 import { EndToEndTestsScreen } from 'sentry-react-native-e2e-tests';
+import { LaunchArguments } from "react-native-launch-arguments";
 
 Sentry.init({
   release: '${SENTRY_RELEASE}',
   dist: '${SENTRY_DIST}',
   dsn: 'https://1df17bd4e543fdb31351dee1768bb679@o447951.ingest.sentry.io/5428561',
+  _experiments: {
+    replaysOnErrorSampleRate: LaunchArguments.value().replaysOnErrorSampleRate,
+  },
+  integrations: [
+    Sentry.mobileReplayIntegration(),
+  ],
 });
 `;
 const e2eComponentPatch = '<EndToEndTestsScreen />';

dev-packages/e2e-tests/src/EndToEndTests.tsx

@@ -3,8 +3,6 @@ import * as React from 'react';
 import { Text, View } from 'react-native';
 import { LaunchArguments } from "react-native-launch-arguments";
 
-import { fetchEvent } from './utils/fetchEvent';
-
 const E2E_TESTS_READY_TEXT = 'E2E Tests Ready';
 
 const getSentryAuthToken = ():
@@ -30,28 +28,6 @@ const EndToEndTestsScreen = (): JSX.Element => {
   const [eventId, setEventId] = React.useState<string | null>(null);
   const [error, setError] = React.useState<string>('No error');
 
-  async function assertEventReceived(eventId: string | undefined) {
-    if (!eventId) {
-      setError('Event ID is required');
-      return;
-    }
-
-    const value = getSentryAuthToken();
-    if ('error' in value) {
-      setError(value.error);
-      return;
-    }
-
-    const event = await fetchEvent(eventId, value.token);
-
-    if (event.event_id !== eventId) {
-      setError('Event ID mismatch');
-      return;
-    }
-
-    setEventId(eventId);
-  }
-
   React.useEffect(() => {
     const client: Sentry.ReactNativeClient | undefined = Sentry.getClient();
 
@@ -63,7 +39,7 @@ const EndToEndTestsScreen = (): JSX.Element => {
     // WARNING: This is only for testing purposes.
     // We only do this to render the eventId onto the UI for end to end tests.
     client.getOptions().beforeSend = (e) => {
-      assertEventReceived(e.event_id);
+      setEventId(e.event_id);
       return e;
     };