diff --git a/dev-packages/e2e-tests/maestro/captureException.yml b/dev-packages/e2e-tests/maestro/captureException.yml index d768aae04a..f746c6fc83 100644 --- a/dev-packages/e2e-tests/maestro/captureException.yml +++ b/dev-packages/e2e-tests/maestro/captureException.yml @@ -1,4 +1,5 @@ appId: ${APP_ID} +jsEngine: graaljs --- - runFlow: utils/launchTestAppClear.yml - tapOn: "Capture Exception" diff --git a/dev-packages/e2e-tests/maestro/captureMessage.yml b/dev-packages/e2e-tests/maestro/captureMessage.yml index 40df1a88cd..9c109e95a4 100644 --- a/dev-packages/e2e-tests/maestro/captureMessage.yml +++ b/dev-packages/e2e-tests/maestro/captureMessage.yml @@ -1,4 +1,5 @@ appId: ${APP_ID} +jsEngine: graaljs --- - runFlow: utils/launchTestAppClear.yml - tapOn: "Capture Message" diff --git a/dev-packages/e2e-tests/maestro/captureReplay.yml b/dev-packages/e2e-tests/maestro/captureReplay.yml new file mode 100644 index 0000000000..a7b65612b4 --- /dev/null +++ b/dev-packages/e2e-tests/maestro/captureReplay.yml @@ -0,0 +1,13 @@ +appId: ${APP_ID} +jsEngine: graaljs +--- +- runFlow: + file: utils/launchTestAppClear.yml + env: + replaysOnErrorSampleRate: 1.0 +- tapOn: "Capture Exception" +- runFlow: utils/assertEventIdVisible.yml +- runFlow: + file: utils/assertReplay.yml + when: + platform: iOS diff --git a/dev-packages/e2e-tests/maestro/captureUnhandledPromiseRejection.yml b/dev-packages/e2e-tests/maestro/captureUnhandledPromiseRejection.yml index 292f2593a6..4f24921129 100644 --- a/dev-packages/e2e-tests/maestro/captureUnhandledPromiseRejection.yml +++ b/dev-packages/e2e-tests/maestro/captureUnhandledPromiseRejection.yml @@ -1,4 +1,5 @@ appId: ${APP_ID} +jsEngine: graaljs --- - runFlow: utils/launchTestAppClear.yml - tapOn: "Unhandled Promise Rejection" diff --git a/dev-packages/e2e-tests/maestro/close.yml b/dev-packages/e2e-tests/maestro/close.yml index e233373426..582396af2e 100644 --- a/dev-packages/e2e-tests/maestro/close.yml +++ b/dev-packages/e2e-tests/maestro/close.yml @@ -1,4 +1,5 @@ appId: ${APP_ID} +jsEngine: graaljs --- - runFlow: utils/launchTestAppClear.yml - tapOn: "Close" diff --git a/dev-packages/e2e-tests/maestro/crash.yml b/dev-packages/e2e-tests/maestro/crash.yml index fb76f3529b..c9256949e7 100644 --- a/dev-packages/e2e-tests/maestro/crash.yml +++ b/dev-packages/e2e-tests/maestro/crash.yml @@ -1,4 +1,5 @@ appId: ${APP_ID} +jsEngine: graaljs --- - runFlow: utils/launchTestAppClear.yml - tapOn: "Crash" diff --git a/dev-packages/e2e-tests/maestro/utils/assertEventIdVisible.yml b/dev-packages/e2e-tests/maestro/utils/assertEventIdVisible.yml index f7995a5f6b..f31a206d66 100644 --- a/dev-packages/e2e-tests/maestro/utils/assertEventIdVisible.yml +++ b/dev-packages/e2e-tests/maestro/utils/assertEventIdVisible.yml @@ -1,4 +1,5 @@ appId: ${APP_ID} +jsEngine: graaljs --- - extendedWaitUntil: visible: @@ -8,3 +9,12 @@ appId: ${APP_ID} - copyTextFrom: id: "eventId" - assertTrue: ${maestro.copiedText} + +- runScript: + file: sentryApi.js + env: + fetch: event + id: ${maestro.copiedText} + sentryAuthToken: ${SENTRY_AUTH_TOKEN} + +- assertTrue: ${output.eventId == maestro.copiedText} diff --git a/dev-packages/e2e-tests/maestro/utils/assertReplay.yml b/dev-packages/e2e-tests/maestro/utils/assertReplay.yml new file mode 100644 index 0000000000..b9885b8227 --- /dev/null +++ b/dev-packages/e2e-tests/maestro/utils/assertReplay.yml @@ -0,0 +1,23 @@ +appId: ${APP_ID} +jsEngine: graaljs +--- +- extendedWaitUntil: + visible: + id: "eventId" + timeout: 60_000 # 60 seconds + +- copyTextFrom: + id: "eventId" +- assertTrue: ${maestro.copiedText} + +- runScript: + file: sentryApi.js + env: + fetch: replay + eventId: ${maestro.copiedText} + sentryAuthToken: ${SENTRY_AUTH_TOKEN} + +- assertTrue: ${output.replayId} +- assertTrue: ${output.replayDuration} +- assertTrue: ${output.replaySegments} +- assertTrue: ${output.replayCodec == "ftypmp42"} diff --git a/dev-packages/e2e-tests/maestro/utils/launchTestAppClear.yml b/dev-packages/e2e-tests/maestro/utils/launchTestAppClear.yml index 3216e1ed89..1ea1c765e7 100644 --- a/dev-packages/e2e-tests/maestro/utils/launchTestAppClear.yml +++ b/dev-packages/e2e-tests/maestro/utils/launchTestAppClear.yml @@ -1,9 +1,15 @@ appId: ${APP_ID} +jsEngine: graaljs --- +# Ensure the app is killed, otherwise we may see "INTERNAL: UiAutomation not connected" errors on Android. +# They seem to be casued by a previous test case running for a long time without UI interactions (e.g. runScript). +- killApp + - launchApp: clearState: true arguments: sentryAuthToken: ${SENTRY_AUTH_TOKEN} + replaysOnErrorSampleRate: ${replaysOnErrorSampleRate} - extendedWaitUntil: visible: "E2E Tests Ready" diff --git a/dev-packages/e2e-tests/maestro/utils/sentryApi.js b/dev-packages/e2e-tests/maestro/utils/sentryApi.js new file mode 100644 index 0000000000..39f10ed298 --- /dev/null +++ b/dev-packages/e2e-tests/maestro/utils/sentryApi.js @@ -0,0 +1,80 @@ +const baseUrl = 'https://sentry.io/api/0/projects/sentry-sdks/sentry-react-native'; + +const RETRY_COUNT = 600; +const RETRY_INTERVAL = 1000; +const requestHeaders = { 'Authorization': `Bearer ${sentryAuthToken}` } + +function sleep(ms) { + // TODO reach out to Maestro & GrallJS via GitHub issues. + // return new Promise(resolve => setTimeout(resolve, ms)); + // Instead, we need to do a busy wait. + const until = Date.now() + ms; + while (Date.now() < until) { + // console.log(`Sleeping for ${until - Date.now()} ms`); + try { + http.get('http://127.0.0.1:1'); + } catch (e) { + // Ignore + } + } +} + +function fetchFromSentry(url) { + console.log(`Fetching ${url}`); + let retries = 0; + const shouldRetry = (response) => { + switch (response.status) { + case 200: + return false; + case 403: + throw new Error(`Could not fetch ${url}: ${response.status} | ${response.body}`); + default: + if (retries++ < RETRY_COUNT) { + console.log(`Request failed (HTTP ${response.status}), retrying: ${retries}/${RETRY_COUNT}`); + return true; + } + throw new Error(`Could not fetch ${url} within retry limit: ${response.status} | ${response.body}`); + } + } + + while (true) { + const response = http.get(url, { headers: requestHeaders }) + if (!shouldRetry(response)) { + console.log(`Received HTTP ${response.status}: body length ${response.body.length}`); + return response.body; + } + sleep(RETRY_INTERVAL); + } +}; + +function setOutput(data) { + for (const [key, value] of Object.entries(data)) { + console.log(`Setting output.${key} = '${value}'`); + output[key] = value; + } +} + +// Note: "fetch", "id", "eventId", etc. are script inputs, see for example assertEventIdIVisible.yml +switch (fetch) { + case 'event': { + const data = json(fetchFromSentry(`${baseUrl}/events/${id}/json/`)); + setOutput({ eventId: data.event_id }); + break; + } + case 'replay': { + const event = json(fetchFromSentry(`${baseUrl}/events/${eventId}/json/`)); + const replayId = event._dsc.replay_id.replace(/\-/g, ''); + const replay = json(fetchFromSentry(`${baseUrl}/replays/${replayId}/`)); + const segment = fetchFromSentry(`${baseUrl}/replays/${replayId}/videos/0/`); + + setOutput({ + replayId: replay.data.id, + replayDuration: replay.data.duration, + replaySegments: replay.data.count_segments, + replayCodec: segment.slice(4, 12) + }); + break; + } + default: + throw new Error(`Unknown "fetch" value: '${fetch}'`); +} diff --git a/dev-packages/e2e-tests/patch-scripts/rn.patch.app.js b/dev-packages/e2e-tests/patch-scripts/rn.patch.app.js index c32274bfa9..fcee381601 100755 --- a/dev-packages/e2e-tests/patch-scripts/rn.patch.app.js +++ b/dev-packages/e2e-tests/patch-scripts/rn.patch.app.js @@ -21,11 +21,18 @@ logger.info('Patching RN App.(js|tsx)', args.app); const initPatch = ` import * as Sentry from '@sentry/react-native'; import { EndToEndTestsScreen } from 'sentry-react-native-e2e-tests'; +import { LaunchArguments } from "react-native-launch-arguments"; Sentry.init({ release: '${SENTRY_RELEASE}', dist: '${SENTRY_DIST}', dsn: 'https://1df17bd4e543fdb31351dee1768bb679@o447951.ingest.sentry.io/5428561', + _experiments: { + replaysOnErrorSampleRate: LaunchArguments.value().replaysOnErrorSampleRate, + }, + integrations: [ + Sentry.mobileReplayIntegration(), + ], }); `; const e2eComponentPatch = ''; diff --git a/dev-packages/e2e-tests/src/EndToEndTests.tsx b/dev-packages/e2e-tests/src/EndToEndTests.tsx index d6c38d9961..173286e57b 100644 --- a/dev-packages/e2e-tests/src/EndToEndTests.tsx +++ b/dev-packages/e2e-tests/src/EndToEndTests.tsx @@ -3,8 +3,6 @@ import * as React from 'react'; import { Text, View } from 'react-native'; import { LaunchArguments } from "react-native-launch-arguments"; -import { fetchEvent } from './utils/fetchEvent'; - const E2E_TESTS_READY_TEXT = 'E2E Tests Ready'; const getSentryAuthToken = (): @@ -30,28 +28,6 @@ const EndToEndTestsScreen = (): JSX.Element => { const [eventId, setEventId] = React.useState(null); const [error, setError] = React.useState('No error'); - async function assertEventReceived(eventId: string | undefined) { - if (!eventId) { - setError('Event ID is required'); - return; - } - - const value = getSentryAuthToken(); - if ('error' in value) { - setError(value.error); - return; - } - - const event = await fetchEvent(eventId, value.token); - - if (event.event_id !== eventId) { - setError('Event ID mismatch'); - return; - } - - setEventId(eventId); - } - React.useEffect(() => { const client: Sentry.ReactNativeClient | undefined = Sentry.getClient(); @@ -63,7 +39,7 @@ const EndToEndTestsScreen = (): JSX.Element => { // WARNING: This is only for testing purposes. // We only do this to render the eventId onto the UI for end to end tests. client.getOptions().beforeSend = (e) => { - assertEventReceived(e.event_id); + setEventId(e.event_id); return e; }; diff --git a/dev-packages/e2e-tests/src/utils/fetchEvent.ts b/dev-packages/e2e-tests/src/utils/fetchEvent.ts deleted file mode 100644 index d792f9748e..0000000000 --- a/dev-packages/e2e-tests/src/utils/fetchEvent.ts +++ /dev/null @@ -1,44 +0,0 @@ -import pRetry from 'p-retry'; -import type { Event } from '@sentry/types'; - -const domain = 'sentry.io'; -const eventEndpoint = 'api/0/projects/sentry-sdks/sentry-react-native/events'; - -const RETRY_COUNT = 600; -const FIRST_RETRY_MS = 1_000; -const MAX_RETRY_TIMEOUT = 5_000; - -const fetchEvent = async (eventId: string, authToken: string): Promise => { - const url = `https://${domain}/${eventEndpoint}/${eventId}/json/`; - - const toRetry = async () => { - const response = await fetch(url, { - headers: { - Authorization: `Bearer ${authToken}`, - 'Content-Type': 'application/json', - }, - method: 'GET', - }); - - const json = (await response.json()) as Event; - if (!json.event_id) { - throw new Error('No event ID found in the response'); - } - - return json; - }; - - const response = await pRetry(toRetry, { - retries: RETRY_COUNT, - minTimeout: FIRST_RETRY_MS, - maxTimeout: MAX_RETRY_TIMEOUT, - factor: 2, - onFailedAttempt: e => { - console.log(`Failed attempt ${e.attemptNumber} of ${RETRY_COUNT}: ${e.message}`); - }, - }); - - return response; -}; - -export { fetchEvent };