diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml index 308404b74bc03..ff888c135be90 100644 --- a/.github/workflows/e2ePerformanceTests.yml +++ b/.github/workflows/e2ePerformanceTests.yml @@ -125,6 +125,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Setup Node + uses: Expensify/App/.github/actions/composite/setupNode@main + - name: Make zip directory for everything to send to AWS Device Farm run: mkdir zip @@ -137,7 +140,7 @@ jobs: # The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it - name: Rename baseline APK - run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-baseline.apk" + run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-main.apk" - name: Download delta APK uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b @@ -147,7 +150,7 @@ jobs: path: zip - name: Rename delta APK - run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-compare.apk" + run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-delta.apk" - name: Copy e2e code into zip folder run: cp -r tests/e2e zip @@ -162,44 +165,72 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: us-west-2 - - name: Schedule AWS Device Farm test run + - name: Schedule AWS Device Farm test run on main branch uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b + id: schedule-awsdf-main with: name: App E2E Performance Regression Tests project_arn: ${{ secrets.AWS_PROJECT_ARN }} device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }} - app_file: zip/app-e2eRelease-baseline.apk + app_file: zip/app-e2eRelease-main.apk app_type: ANDROID_APP test_type: APPIUM_NODE test_package_file: App.zip test_package_type: APPIUM_NODE_TEST_PACKAGE - test_spec_file: tests/e2e/TestSpec.yml + test_spec_file: tests/e2e/TestSpecMain.yml test_spec_type: APPIUM_NODE_TEST_SPEC remote_src: false file_artifacts: Customer Artifacts.zip + log_artifacts: debug.log cleanup: true - - name: Unzip AWS Device Farm results - if: ${{ always() }} - run: unzip "Customer Artifacts.zip" - - - name: Print AWS Device Farm run results - if: ${{ always() }} - run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md" - - - name: Print AWS Device Farm verbose run results - if: ${{ always() && runner.debug != null && fromJSON(runner.debug) }} - run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/debug.log" - -# TODO: Once tests are more reliable we should uncomment this -# - name: Check if test failed, if so post the results and add the DeployBlocker label -# run: | -# if grep -q '🔴' ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md; then -# gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash -# gh pr comment ${{ inputs.PR_NUMBER }} -F ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md -# gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker." -# else -# echo '✅ no performance regression detected' -# fi -# env: -# GITHUB_TOKEN: ${{ github.token }} + - name: Print logs if run failed + if: failure() + run: | + echo ${{ steps.schedule-awsdf-main.outputs.data }} + unzip "Customer Artifacts.zip" -d mainResults + cat ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/debug.log + + - name: Unzip AWS Device Farm main results + run: unzip "Customer Artifacts.zip" -d mainResults + + - name: Delete Customer Artifacts.zip + run: rm "Customer Artifacts.zip" + + - name: Schedule AWS Device Farm test run on delta branch + uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b + with: + name: App E2E Performance Regression Tests + project_arn: ${{ secrets.AWS_PROJECT_ARN }} + device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }} + app_file: zip/app-e2eRelease-delta.apk + app_type: ANDROID_APP + test_type: APPIUM_NODE + test_package_file: App.zip + test_package_type: APPIUM_NODE_TEST_PACKAGE + test_spec_file: tests/e2e/TestSpecDelta.yml + test_spec_type: APPIUM_NODE_TEST_SPEC + remote_src: false + file_artifacts: Customer Artifacts.zip + cleanup: true + + - name: Unzip AWS Device Farm delta results + run: unzip "Customer Artifacts.zip" -d deltaResults + + - name: Compare results + run: node tests/e2e/merge.js --mainPath ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/main.json --deltaPath ./deltaResults//Host_Machine_Files/\$WORKING_DIRECTORY/delta.json --outputPath ./output.md + + - name: Print results + run: cat "./output.md" + + - name: Check if test failed, if so post the results and add the DeployBlocker label + run: | + if grep -q '🔴' ./output.md; then + gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash + gh pr comment ${{ inputs.PR_NUMBER }} -F ./output.md + gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker." + else + echo '✅ no performance regression detected' + fi + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/metro.config.js b/metro.config.js index bf2ff904df702..62ca2a25c6b24 100644 --- a/metro.config.js +++ b/metro.config.js @@ -7,9 +7,10 @@ require('dotenv').config(); const defaultConfig = getDefaultConfig(__dirname); const isUsingMockAPI = process.env.E2E_TESTING === 'true'; + if (isUsingMockAPI) { // eslint-disable-next-line no-console - console.warn('⚠️ Using mock API'); + console.log('⚠️⚠️⚠️⚠️ Using mock API ⚠️⚠️⚠️⚠️'); } /** @@ -25,10 +26,14 @@ const config = { resolveRequest: (context, moduleName, platform) => { const resolution = context.resolveRequest(context, moduleName, platform); if (isUsingMockAPI && moduleName.includes('/API')) { + const originalPath = resolution.filePath; + const mockPath = originalPath.replace('src/libs/API.ts', 'src/libs/E2E/API.mock.js').replace('/src/libs/API.js/', 'src/libs/E2E/API.mock.js'); + // eslint-disable-next-line no-console + console.log('⚠️⚠️⚠️⚠️ Replacing resolution path', originalPath, ' => ', mockPath); + return { ...resolution, - // TODO: Change API.mock.js extension once it is migrated to TypeScript - filePath: resolution.filePath.replace(/src\/libs\/API.js/, 'src/libs/E2E/API.mock.js'), + filePath: mockPath, }; } return resolution; diff --git a/package.json b/package.json index 993b8d165ed08..7d540c0d35069 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,9 @@ "analyze-packages": "ANALYZE_BUNDLE=true webpack --config config/webpack/webpack.common.js --env envFile=.env.production", "symbolicate:android": "npx metro-symbolicate android/app/build/generated/sourcemaps/react/release/index.android.bundle.map", "symbolicate:ios": "npx metro-symbolicate main.jsbundle.map", - "test:e2e": "node tests/e2e/testRunner.js --development", + "test:e2e:main": "node tests/e2e/testRunner.js --development --branch main --skipCheckout", + "test:e2e:delta": "node tests/e2e/testRunner.js --development --branch main --label delta --skipCheckout", + "test:e2e:compare": "node tests/e2e/merge.js", "gh-actions-unused-styles": "./.github/scripts/findUnusedKeys.sh", "workflow-test": "./workflow_tests/scripts/runWorkflowTests.sh", "workflow-test:generate": "node workflow_tests/utils/preGenerateTest.js" diff --git a/tests/e2e/TestSpecDelta.yml b/tests/e2e/TestSpecDelta.yml new file mode 100644 index 0000000000000..2d4906855ca8c --- /dev/null +++ b/tests/e2e/TestSpecDelta.yml @@ -0,0 +1,26 @@ +version: 0.1 + +phases: + install: + commands: + # Install correct version of node + - export NVM_DIR=$HOME/.nvm + - . $NVM_DIR/nvm.sh + - nvm install 16.15.1 + - nvm use 16.15.1 + + # Reverse ports using AWS magic + - PORT=4723 + - IP_ADDRESS=$(ip -4 addr show eth0 | grep -Po "(?<=inet\s)\d+(\.\d+){3}") + - reverse_values="{\"ip_address\":\"$IP_ADDRESS\",\"local_port\":\"$PORT\",\"remote_port\":\"$PORT\"}" + - "curl -H \"Content-Type: application/json\" -X POST -d \"$reverse_values\" http://localhost:31007/reverse_forward_tcp" + - adb reverse tcp:$PORT tcp:$PORT + + test: + commands: + - cd zip + - npm install underscore + - node e2e/testRunner.js -- --skipInstallDeps --buildMode "skip" --skipCheckout --label delta --appPath app-e2eRelease-delta.apk + +artifacts: +- $WORKING_DIRECTORY diff --git a/tests/e2e/TestSpec.yml b/tests/e2e/TestSpecMain.yml similarity index 91% rename from tests/e2e/TestSpec.yml rename to tests/e2e/TestSpecMain.yml index 6ea1d9570ae9d..6cf1c5d0b2734 100644 --- a/tests/e2e/TestSpec.yml +++ b/tests/e2e/TestSpecMain.yml @@ -20,7 +20,7 @@ phases: commands: - cd zip - npm install underscore - - node e2e/testRunner.js -- --skipInstallDeps --buildMode "skip" --skipCheckout + - node e2e/testRunner.js -- --skipInstallDeps --buildMode "skip" --skipCheckout --branch main --appPath app-e2eRelease-main.apk artifacts: - $WORKING_DIRECTORY diff --git a/tests/e2e/compare/compare.js b/tests/e2e/compare/compare.js index 2929b03f4d708..9d40ab2bf312a 100644 --- a/tests/e2e/compare/compare.js +++ b/tests/e2e/compare/compare.js @@ -1,7 +1,6 @@ const fs = require('fs/promises'); const fsSync = require('fs'); const _ = require('underscore'); -const {OUTPUT_DIR} = require('../config'); const {computeProbability, computeZ} = require('./math'); const printToConsole = require('./output/console'); const writeToMarkdown = require('./output/markdown'); @@ -119,7 +118,7 @@ function compareResults(compareEntries, baselineEntries) { }; } -module.exports = (baselineFile = `${OUTPUT_DIR}/baseline.json`, compareFile = `${OUTPUT_DIR}/compare.json`, outputFormat = 'all') => { +module.exports = (baselineFile, compareFile, outputFile, outputFormat = 'all') => { const hasBaselineFile = fsSync.existsSync(baselineFile); if (!hasBaselineFile) { throw new Error(`Baseline results files "${baselineFile}" does not exists.`); @@ -136,7 +135,7 @@ module.exports = (baselineFile = `${OUTPUT_DIR}/baseline.json`, compareFile = `$ printToConsole(outputData); } if (outputFormat === 'markdown' || outputFormat === 'all') { - return writeToMarkdown(`${OUTPUT_DIR}/output.md`, outputData); + return writeToMarkdown(outputFile, outputData); } }); }); diff --git a/tests/e2e/config.js b/tests/e2e/config.js index 4f08754cfec28..3b1856ab8ad85 100644 --- a/tests/e2e/config.js +++ b/tests/e2e/config.js @@ -23,10 +23,7 @@ const TEST_NAMES = { module.exports = { APP_PACKAGE: 'com.expensify.chat.adhoc', - APP_PATHS: { - baseline: './app-e2eRelease-baseline.apk', - compare: './app-e2eRelease-compare.apk', - }, + APP_PATH: './app-e2eRelease-main.apk', ENTRY_FILE: 'src/libs/E2E/reactNativeLaunchingTest.js', @@ -34,13 +31,10 @@ module.exports = { SERVER_PORT: 4723, // The amount of times a test should be executed for average performance metrics - RUNS: 90, + RUNS: 60, DEFAULT_BASELINE_BRANCH: 'main', - // The amount of runs that should happen without counting test results - WARM_UP_RUNS: 3, - OUTPUT_DIR, // The file to write intermediate results to diff --git a/tests/e2e/config.local.js b/tests/e2e/config.local.js index 0c38c3f1056f1..15b091d8ba70f 100644 --- a/tests/e2e/config.local.js +++ b/tests/e2e/config.local.js @@ -1,10 +1,5 @@ module.exports = { - APP_PACKAGE: 'com.expensify.chat.dev', - - WARM_UP_RUNS: 1, + APP_PACKAGE: 'com.expensify.chat.adhoc', + APP_PATH: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', RUNS: 8, - APP_PATHS: { - baseline: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', - compare: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', - }, }; diff --git a/tests/e2e/merge.js b/tests/e2e/merge.js new file mode 100644 index 0000000000000..0ee4761373158 --- /dev/null +++ b/tests/e2e/merge.js @@ -0,0 +1,27 @@ +const compare = require('./compare/compare'); +const {OUTPUT_DIR} = require('./config'); + +const args = process.argv.slice(2); + +let mainPath = `${OUTPUT_DIR}/main.json`; +if (args.includes('--mainPath')) { + mainPath = args[args.indexOf('--mainPath') + 1]; +} + +let deltaPath = `${OUTPUT_DIR}/delta.json`; +if (args.includes('--deltaPath')) { + deltaPath = args[args.indexOf('--deltaPath') + 1]; +} + +let outputPath = `${OUTPUT_DIR}/output.md`; +if (args.includes('--outputPath')) { + outputPath = args[args.indexOf('--outputPath') + 1]; +} + +async function run() { + await compare(mainPath, deltaPath, outputPath, 'all'); + + process.exit(0); +} + +run(); diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index 2a5aee78715fc..5c6c33bdf7e91 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -1,16 +1,22 @@ /** - * The test runner takes care of running the e2e tests. - * It will run the tests twice. Once on the branch that - * we want to base the results on (e.g. main), and then - * again on another branch we want to compare against the - * base (e.g. a new feature branch). + * Multifaceted script, its main function is running the e2e tests. + * + * When running in a local environment it can take care of building the APKs required for e2e testing + * When running on the CI (depending on the flags passed to it) it will skip building and just package/re-sign + * the correct e2e JS bundle into an existing APK + * + * It will run only one set of tests per branch, for you to compare results to get a performance analysis + * You need to run it twice, once with the base branch (--branch main) and another time with another branch + * and a label to (--branch my_branch --label delta) + * + * This two runs will generate a main.json and a delta.json with the performance data, which then you can merge via + * node tests/e2e/merge.js */ /* eslint-disable @lwc/lwc/no-async-await,no-restricted-syntax,no-await-in-loop */ const fs = require('fs'); const _ = require('underscore'); const defaultConfig = require('./config'); -const compare = require('./compare/compare'); const Logger = require('./utils/logger'); const execAsync = require('./utils/execAsync'); const killApp = require('./utils/killApp'); @@ -21,10 +27,20 @@ const math = require('./measure/math'); const writeTestStats = require('./measure/writeTestStats'); const withFailTimeout = require('./utils/withFailTimeout'); const reversePort = require('./utils/androidReversePort'); -const getCurrentBranchName = require('./utils/getCurrentBranchName'); +// VARIABLE CONFIGURATION const args = process.argv.slice(2); +let branch = 'main'; +if (args.includes('--branch')) { + branch = args[args.indexOf('--branch') + 1]; +} + +let label = branch; +if (args.includes('--label')) { + label = args[args.indexOf('--label') + 1]; +} + let config = defaultConfig; const setConfigPath = (configPathParam) => { let configPath = configPathParam; @@ -35,12 +51,14 @@ const setConfigPath = (configPathParam) => { config = _.extend(defaultConfig, customConfig); }; -let baselineBranch = process.env.baseline || config.DEFAULT_BASELINE_BRANCH; +const skipCheckout = args.includes('--skipCheckout'); + +const skipInstallDeps = args.includes('--skipInstallDeps'); // There are three build modes: // 1. full: rebuilds the full native app in (e2e) release mode // 2. js-only: only rebuilds the js bundle, and then re-packages -// the existing native app with the new package. If there +// the existing native app with the new bundle. If there // is no existing native app, it will fallback to mode "full" // 3. skip: does not rebuild anything, and just runs the existing native app let buildMode = 'full'; @@ -49,28 +67,51 @@ let buildMode = 'full'; const isDevMode = args.includes('--development'); if (isDevMode) { setConfigPath('config.local.js'); - baselineBranch = getCurrentBranchName(); buildMode = 'js-only'; } +if (args.includes('--buildMode')) { + buildMode = args[args.indexOf('--buildMode') + 1]; +} + if (args.includes('--config')) { const configPath = args[args.indexOf('--config') + 1]; setConfigPath(configPath); } -// Clear all files from previous jobs -try { - fs.rmSync(config.OUTPUT_DIR, {recursive: true, force: true}); - fs.mkdirSync(config.OUTPUT_DIR); -} catch (error) { - // Do nothing - console.error(error); +// Important set app path after correct config file has been set +let appPath = config.APP_PATH; +if (args.includes('--appPath')) { + appPath = args[args.indexOf('--appPath') + 1]; } +// Create some variables after the correct config file has been loaded +const OUTPUT_FILE = `${config.OUTPUT_DIR}/${label}.json`; + if (isDevMode) { - Logger.note(`Running in development mode. Set baseline branch to same as current ${baselineBranch}`); + Logger.note(`🟠 Running in development mode.`); +} + +if (isDevMode) { + // On dev mode only delete any existing output file but keep the folder + if (fs.existsSync(OUTPUT_FILE)) { + fs.rmSync(OUTPUT_FILE); + } +} else { + // On CI it is important to re-create the output dir, it has a different owner + // therefore this process cannot write to it + try { + fs.rmSync(config.OUTPUT_DIR, {recursive: true, force: true}); + + fs.mkdirSync(config.OUTPUT_DIR); + } catch (error) { + // Do nothing + console.error(error); + } } +// START OF TEST CODE + const restartApp = async () => { Logger.log('Killing app …'); await killApp('android', config.APP_PACKAGE); @@ -78,12 +119,7 @@ const restartApp = async () => { await launchApp('android', config.APP_PACKAGE); }; -const runTestsOnBranch = async (baselineOrCompare, branch) => { - if (args.includes('--buildMode')) { - buildMode = args[args.indexOf('--buildMode') + 1]; - } - let appPath = baselineOrCompare === 'baseline' ? config.APP_PATHS.baseline : config.APP_PATHS.compare; - +const runTests = async () => { // check if using buildMode "js-only" or "none" is possible if (buildMode !== 'full') { const appExists = fs.existsSync(appPath); @@ -95,30 +131,29 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { } } - if (branch != null) { + if (branch != null && !skipCheckout) { // Switch branch - Logger.log(`Preparing ${baselineOrCompare} tests on branch '${branch}'`); + Logger.log(`Preparing tests on branch '${branch}' - git checkout`); await execAsync(`git checkout ${branch}`); } - if (!args.includes('--skipInstallDeps')) { - Logger.log(`Preparing ${baselineOrCompare} tests on branch '${branch}' - npm install`); + if (!skipInstallDeps) { + Logger.log(`Preparing tests on branch '${branch}' - npm install`); await execAsync('npm i'); } // Build app if (buildMode === 'full') { - Logger.log(`Preparing ${baselineOrCompare} tests on branch '${branch}' - building app`); + Logger.log(`Preparing tests on branch '${branch}' - building app`); await execAsync('npm run android-build-e2e'); } else if (buildMode === 'js-only') { - Logger.log(`Preparing ${baselineOrCompare} tests on branch '${branch}' - building js bundle`); + Logger.log(`Preparing tests on branch '${branch}' - building js bundle`); // Build a new JS bundle const tempDir = `${config.OUTPUT_DIR}/temp`; const tempBundlePath = `${tempDir}/index.android.bundle`; await execAsync(`rm -rf ${tempDir} && mkdir ${tempDir}`); - await execAsync(`E2E_TESTING=true npx react-native bundle --platform android --dev false --entry-file ${config.ENTRY_FILE} --bundle-output ${tempBundlePath}`); - + await execAsync(`npx react-native bundle --platform android --dev false --entry-file ${config.ENTRY_FILE} --bundle-output ${tempBundlePath}`, {E2E_TESTING: 'true'}); // Repackage the existing native app with the new bundle const tempApkPath = `${tempDir}/app-release.apk`; await execAsync(`./scripts/android-repackage-app-bundle-and-sign.sh ${appPath} ${tempBundlePath} ${tempApkPath}`); @@ -169,30 +204,29 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { server.setTestConfig(testConfig); - const warmupLogs = Logger.progressInfo(`Running test '${testConfig.name}'`); - for (let warmUpRuns = 0; warmUpRuns < config.WARM_UP_RUNS; warmUpRuns++) { - const progressText = `(${testIndex + 1}/${numOfTests}) Warmup for test '${testConfig.name}' (iteration ${warmUpRuns + 1}/${config.WARM_UP_RUNS})`; - warmupLogs.updateText(progressText); + const warmupLogs = Logger.progressInfo(`Running warmup '${testConfig.name}'`); - await restartApp(); + let progressText = `Warmup for suite '${testConfig.name}' [${testIndex + 1}/${numOfTests}]\n`; + warmupLogs.updateText(progressText); + + await restartApp(); + + await withFailTimeout( + new Promise((resolve) => { + const cleanup = server.addTestDoneListener(() => { + cleanup(); + resolve(); + }); + }), + progressText, + ); - await withFailTimeout( - new Promise((resolve) => { - const cleanup = server.addTestDoneListener(() => { - Logger.log(`Warmup ${warmUpRuns + 1} done!`); - cleanup(); - resolve(); - }); - }), - progressText, - ); - } warmupLogs.done(); // We run each test multiple time to average out the results const testLog = Logger.progressInfo(''); for (let i = 0; i < config.RUNS; i++) { - const progressText = `(${testIndex + 1}/${numOfTests}) Running test '${testConfig.name}' (iteration ${i + 1}/${config.RUNS})`; + progressText = `Suite '${testConfig.name}' [${testIndex + 1}/${numOfTests}], iteration [${i + 1}/${config.RUNS}]\n`; testLog.updateText(progressText); await restartApp(); @@ -220,7 +254,7 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { // Calculate statistics and write them to our work file progressLog = Logger.progressInfo('Calculating statics and writing results'); - const outputFileName = `${config.OUTPUT_DIR}/${baselineOrCompare}.json`; + for (const testName of _.keys(durationsByTestName)) { const stats = math.getStats(durationsByTestName[testName]); await writeTestStats( @@ -228,7 +262,7 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { name: testName, ...stats, }, - outputFileName, + OUTPUT_FILE, ); } progressLog.done(); @@ -236,32 +270,24 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { await server.stop(); }; -const runTests = async () => { +const run = async () => { Logger.info('Running e2e tests'); try { - const skipCheckout = args.includes('--skipCheckout'); - - // Run tests on baseline branch - await runTestsOnBranch('baseline', skipCheckout ? null : baselineBranch); - - // Run tests on current branch - await runTestsOnBranch('compare', skipCheckout ? null : '-'); - - await compare(); + await runTests(); process.exit(0); } catch (e) { Logger.info('\n\nE2E test suite failed due to error:', e, '\nPrinting full logs:\n\n'); // Write logcat, meminfo, emulator info to file as well: - require('node:child_process').execSync(`adb logcat -d > ${config.OUTPUT_DIR}/logcat.txt`); - require('node:child_process').execSync(`adb shell "cat /proc/meminfo" > ${config.OUTPUT_DIR}/meminfo.txt`); - require('node:child_process').execSync(`adb shell "getprop" > ${config.OUTPUT_DIR}/emulator-properties.txt`); + require('child_process').execSync(`adb logcat -d > ${config.OUTPUT_DIR}/logcat.txt`); + require('child_process').execSync(`adb shell "cat /proc/meminfo" > ${config.OUTPUT_DIR}/meminfo.txt`); + require('child_process').execSync(`adb shell "getprop" > ${config.OUTPUT_DIR}/emulator-properties.txt`); - require('node:child_process').execSync(`cat ${config.LOG_FILE}`); + require('child_process').execSync(`cat ${config.LOG_FILE}`); try { - require('node:child_process').execSync(`cat ~/.android/avd/${process.env.AVD_NAME || 'test'}.avd/config.ini > ${config.OUTPUT_DIR}/emulator-config.ini`); + require('child_process').execSync(`cat ~/.android/avd/${process.env.AVD_NAME || 'test'}.avd/config.ini > ${config.OUTPUT_DIR}/emulator-config.ini`); } catch (ignoredError) { // the error is ignored, as the file might not exist if the test // run wasn't started with an emulator @@ -270,4 +296,4 @@ const runTests = async () => { } }; -runTests(); +run(); diff --git a/tests/e2e/utils/execAsync.js b/tests/e2e/utils/execAsync.js index c51a328e914d6..be80452c8acb0 100644 --- a/tests/e2e/utils/execAsync.js +++ b/tests/e2e/utils/execAsync.js @@ -1,32 +1,39 @@ -const {exec} = require('node:child_process'); +const {exec} = require('child_process'); const Logger = require('./logger'); /** * Executes a command none-blocking by wrapping it in a promise. * In addition to the promise it returns an abort function. * @param {string} command + * @param {object} env environment variables * @returns {Promise} */ -module.exports = (command) => { - let process; +module.exports = (command, env = {}) => { + let childProcess; const promise = new Promise((resolve, reject) => { - Logger.log('Output of command:', command); - process = exec( + const finalEnv = { + ...process.env, + ...env, + }; + + Logger.important(command); + + childProcess = exec( command, { - encoding: 'utf8', maxBuffer: 1024 * 1024 * 10, // Increase max buffer to 10MB, to avoid errors + env: finalEnv, }, (error, stdout) => { if (error) { if (error && error.killed) { resolve(); } else { - Logger.log(`failed with error: ${error}`); + Logger.error(`failed with error: ${error}`); reject(error); } } else { - Logger.log(stdout); + Logger.note(stdout); resolve(stdout); } }, @@ -34,7 +41,7 @@ module.exports = (command) => { }); promise.abort = () => { - process.kill('SIGINT'); + childProcess.kill('SIGINT'); }; return promise; diff --git a/tests/e2e/utils/getCurrentBranchName.js b/tests/e2e/utils/getCurrentBranchName.js index 3380bd23ef153..ca2f0cba97b08 100644 --- a/tests/e2e/utils/getCurrentBranchName.js +++ b/tests/e2e/utils/getCurrentBranchName.js @@ -1,4 +1,4 @@ -const {execSync} = require('node:child_process'); +const {execSync} = require('child_process'); const getCurrentBranchName = () => { const stdout = execSync('git rev-parse --abbrev-ref HEAD', { diff --git a/tests/e2e/utils/logger.js b/tests/e2e/utils/logger.js index 1f2fff315bfc1..7da1e8330bfc6 100644 --- a/tests/e2e/utils/logger.js +++ b/tests/e2e/utils/logger.js @@ -1,4 +1,5 @@ const fs = require('fs'); +const path = require('path'); const {LOG_FILE} = require('../config'); let isVerbose = true; @@ -12,6 +13,9 @@ const LOGGER_PROGRESS_REFRESH_RATE = process.env.LOGGER_PROGRESS_REFRESH_RATE || const COLOR_DIM = '\x1b[2m'; const COLOR_RESET = '\x1b[0m'; const COLOR_YELLOW = '\x1b[33m'; +const COLOR_RED = '\x1b[31m'; +const COLOR_BLUE = '\x1b[34m'; +const COLOR_GREEN = '\x1b[32m'; const log = (...args) => { if (isVerbose) { @@ -20,6 +24,12 @@ const log = (...args) => { // Write to log file if (!fs.existsSync(LOG_FILE)) { + // Check that the directory exists + const logDir = path.dirname(LOG_FILE); + if (!fs.existsSync(logDir)) { + fs.mkdirSync(logDir); + } + fs.writeFileSync(LOG_FILE, ''); } const time = new Date(); @@ -27,6 +37,35 @@ const log = (...args) => { fs.appendFileSync(LOG_FILE, `[${timeStr}] ${args.join(' ')}\n`); }; +const info = (...args) => { + log('> ', ...args); +}; + +const important = (...args) => { + const lines = [`🟦 ${COLOR_BLUE}`, ...args, `${COLOR_RESET}\n`]; + log(...lines); +}; + +const success = (...args) => { + const lines = [`🟦 ${COLOR_GREEN}`, ...args, `${COLOR_RESET}\n`]; + log(...lines); +}; + +const warn = (...args) => { + const lines = [`\n${COLOR_YELLOW}⚠️`, ...args, `${COLOR_RESET}\n`]; + log(...lines); +}; + +const note = (...args) => { + const lines = [`${COLOR_DIM}`, ...args, `${COLOR_RESET}\n`]; + log(...lines); +}; + +const error = (...args) => { + const lines = [`\n🔴 ${COLOR_RED}`, ...args, `${COLOR_RESET}\n`]; + log(...lines); +}; + const progressInfo = (textParam) => { let text = textParam || ''; const getTexts = () => [`🕛 ${text}`, `🕔 ${text}`, `🕗 ${text}`, `🕙 ${text}`]; @@ -51,34 +90,23 @@ const progressInfo = (textParam) => { }, done: () => { clearInterval(timer); - process.stdout.write(`\r✅ ${text} ${getTimeText()}\n`); + success(`\r✅ ${text} ${getTimeText()}\n`); }, error: () => { clearInterval(timer); - process.stdout.write(`\r❌ ${text} ${getTimeText()}\n`); + error(`\r❌ ${text} ${getTimeText()}\n`); }, }; }; -const info = (...args) => { - log('> ', ...args); -}; - -const warn = (...args) => { - const lines = [`\n${COLOR_YELLOW}⚠️`, ...args, `${COLOR_RESET}\n`]; - log(...lines); -}; - -const note = (...args) => { - const lines = [`\n💡${COLOR_DIM}`, ...args, `${COLOR_RESET}\n`]; - log(...lines); -}; - module.exports = { log, info, warn, note, + error, + success, + important, progressInfo, setLogLevelVerbose, };