diff --git a/.github/workflows/perf-tests.yml b/.github/workflows/perf-tests.yml new file mode 100644 index 0000000..b9d0dc9 --- /dev/null +++ b/.github/workflows/perf-tests.yml @@ -0,0 +1,99 @@ +name: Performance Tests +on: + push: + branches-ignore: + - 'main-built' + pull_request: + +jobs: + performance-tests: + name: 'Performance Tests' + runs-on: ubuntu-latest + env: + WP_BASE_URL: 'http://localhost:8888' + WP_USERNAME: 'admin' + WP_PASSWORD: 'password' + WP_AUTH_STORAGE: '.auth/wordpress.json' + WP_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: yarn + + - name: Cache Composer dependencies + uses: actions/cache@v4 + with: + path: /tmp/composer-cache + key: ${{ runner.os }}-${{ hashFiles('**/composer.lock') }} + + - name: Setup composer + uses: php-actions/composer@v6 + with: + php_version: '8.3' + dev: no + + - name: Install dependencies + run: yarn install --immutable + + - name: Build packages + run: yarn build + + - name: Playwright install + run: yarn playwright install chromium + + - name: Start wp-env + run: yarn wp-env + + - name: Run tests + run: | + yarn test:performance + mv ${{ env.WP_ARTIFACTS_PATH }}/performance-results.json ${{ runner.temp }}/results_after.json + + - name: Check out base commit + run: | + if [[ -z "$BASE_REF" ]]; then + git fetch -n origin $BASE_SHA + git reset --hard $BASE_SHA + else + git fetch -n origin $BASE_REF + git reset --hard $BASE_SHA + fi + env: + BASE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || '' }} + BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} + + # Run tests without causing job to fail if they don't pass (e.g. because of env issues). + - name: Run tests for base + run: | + npm run test:performance || true + if [ -f "{{ env.WP_ARTIFACTS_PATH }}/performance-results.json" ]; then + mv ${{ env.WP_ARTIFACTS_PATH }}/performance-results.json ${{ runner.temp }}/results_before.json + fi; + + - name: Reset to original commit + run: | + git reset --hard $GITHUB_SHA + + - name: Compare results with base + run: | + if [ -f "${{ runner.temp }}/results_before.json" ]; then + yarn test:performance:results ${{ runner.temp }}/results_after.json ${{ runner.temp }}/results_before.json + else + yarn test:performance:results ${{ runner.temp }}/results_after.json + fi; + + - name: Add workflow summary + run: | + cat ${{ env.WP_ARTIFACTS_PATH }}/performance-results.md >> $GITHUB_STEP_SUMMARY + + - name: Upload performance results + if: success() + uses: actions/upload-artifact@v4 + with: + name: performance-results + path: ${{ env.WP_ARTIFACTS_PATH }}/performance-results.json diff --git a/jest.config.js b/jest.config.js index 7ebd5f5..4b2af1a 100644 --- a/jest.config.js +++ b/jest.config.js @@ -5,7 +5,10 @@ module.exports = { }, modulePathIgnorePatterns: ['/vendor/'], testEnvironment: 'jsdom', - testPathIgnorePatterns: ['/tests/e2e/'], + testPathIgnorePatterns: [ + '/tests/e2e/', + '/tests/performance/', + ], collectCoverage: true, coverageReporters: ['text', 'cobertura'], }; diff --git a/package.json b/package.json index 977ee0f..784ec25 100644 --- a/package.json +++ b/package.json @@ -1,69 +1,72 @@ { - "name": "codeb-feature-flags", - "version": "0.3.2", - "description": "Allows developers to enable / disable features based on flags.", - "license": "ISC", - "author": "Mohan Raj ", - "scripts": { - "build": "wp-scripts build", - "lint:css": "wp-scripts lint-style", - "lint:css:fix": "npm run lint:css -- --fix", - "lint:js": "wp-scripts lint-js", - "lint:js:fix": "wp-scripts lint-js --fix", - "prepare": "husky", - "start": "wp-scripts start", - "test:e2e": "wp-scripts test-playwright", - "test:js": "wp-scripts test-unit-js", - "test:watch": "wp-scripts test-unit-js --watch", - "version:major": "node ./scripts/version major", - "version:minor": "node ./scripts/version minor", - "version:patch": "node ./scripts/version patch", - "wp-env": "wp-env start", - "wp-env:coverage": "wp-env start --xdebug=coverage", - "php:unit": "wp-env run --env-cwd='wp-content/plugins/feature-flags' tests-wordpress composer test:unit", - "php:integration": "wp-env run tests-wordpress --env-cwd=wp-content/plugins/feature-flags composer test:integration", - "php:multisite": "wp-env run tests-wordpress --env-cwd=wp-content/plugins/feature-flags composer test:multisite" - }, - "dependencies": { - "@testing-library/user-event": "^14.5.2", - "@wordpress/api-fetch": "^6.48.0", - "@wordpress/components": "^27.1.0", - "@wordpress/data": "^9.23.0", - "@wordpress/dom-ready": "^3.53.0", - "@wordpress/hooks": "^3.53.0", - "@wordpress/i18n": "^4.53.0", - "@wordpress/notices": "^4.21.0", - "dotenv": "^16.4.5", - "react": "18.2.0", - "react-dom": "18.2.0", - "react-syntax-highlighter": "^15.5.0", - "react-test-renderer": "^18.2.0", - "ts-loader": "^9.5.1", - "typescript": "^5.4.2" - }, - "devDependencies": { - "@playwright/test": "^1.42.1", - "@testing-library/jest-dom": "^6.4.2", - "@testing-library/react": "14.2.1", - "@types/jest": "^29.5.12", - "@types/node": "^20.11.25", - "@types/react-syntax-highlighter": "^15.5.11", - "@types/wordpress__components": "^23.0.11", - "@wordpress/e2e-test-utils-playwright": "^0.21.0", - "@wordpress/env": "^9.5.0", - "@wordpress/eslint-plugin": "^17.10.0", - "@wordpress/scripts": "^27.4.0", - "eslint": "^8.57.0", - "eslint-import-resolver-alias": "^1.1.2", - "eslint-plugin-cypress": "^2.15.1", - "eslint-plugin-import": "^2.29.1", - "husky": "^9.0.11", - "jest-environment-jsdom": "^29.7.0", - "prettier": "^3.2.5" - }, - "keywords": [ - "feature flags", - "wordpress", - "plugin" - ] + "name": "codeb-feature-flags", + "version": "0.3.2", + "description": "Allows developers to enable / disable features based on flags.", + "license": "ISC", + "author": "Mohan Raj ", + "scripts": { + "build": "wp-scripts build", + "lint:css": "wp-scripts lint-style", + "lint:css:fix": "npm run lint:css -- --fix", + "lint:js": "wp-scripts lint-js", + "lint:js:fix": "wp-scripts lint-js --fix", + "prepare": "husky", + "start": "wp-scripts start", + "test:e2e": "wp-scripts test-playwright", + "test:js": "wp-scripts test-unit-js", + "test:performance": "wp-scripts test-playwright --config tests/performance/playwright.config.ts", + "test:performance:merge-reports": "playwright merge-reports --reporter tests/performance/config/performance-reporter.ts ./blob-report", + "test:performance:results": "node tests/performance/cli/results.js", + "test:watch": "wp-scripts test-unit-js --watch", + "version:major": "node ./scripts/version major", + "version:minor": "node ./scripts/version minor", + "version:patch": "node ./scripts/version patch", + "wp-env": "wp-env start", + "wp-env:coverage": "wp-env start --xdebug=coverage", + "php:unit": "wp-env run --env-cwd='wp-content/plugins/feature-flags' tests-wordpress composer test:unit", + "php:integration": "wp-env run tests-wordpress --env-cwd=wp-content/plugins/feature-flags composer test:integration", + "php:multisite": "wp-env run tests-wordpress --env-cwd=wp-content/plugins/feature-flags composer test:multisite" + }, + "dependencies": { + "@testing-library/user-event": "^14.5.2", + "@wordpress/api-fetch": "^6.48.0", + "@wordpress/components": "^27.1.0", + "@wordpress/data": "^9.23.0", + "@wordpress/dom-ready": "^3.53.0", + "@wordpress/hooks": "^3.53.0", + "@wordpress/i18n": "^4.53.0", + "@wordpress/notices": "^4.21.0", + "dotenv": "^16.4.5", + "react": "18.2.0", + "react-dom": "18.2.0", + "react-syntax-highlighter": "^15.5.0", + "react-test-renderer": "^18.2.0", + "ts-loader": "^9.5.1", + "typescript": "^5.4.2" + }, + "devDependencies": { + "@playwright/test": "^1.42.1", + "@testing-library/jest-dom": "^6.4.2", + "@testing-library/react": "14.2.1", + "@types/jest": "^29.5.12", + "@types/node": "^20.11.25", + "@types/react-syntax-highlighter": "^15.5.11", + "@types/wordpress__components": "^23.0.11", + "@wordpress/e2e-test-utils-playwright": "^0.21.0", + "@wordpress/env": "^9.5.0", + "@wordpress/eslint-plugin": "^17.10.0", + "@wordpress/scripts": "^27.4.0", + "eslint": "^8.57.0", + "eslint-import-resolver-alias": "^1.1.2", + "eslint-plugin-cypress": "^2.15.1", + "eslint-plugin-import": "^2.29.1", + "husky": "^9.0.11", + "jest-environment-jsdom": "^29.7.0", + "prettier": "^3.2.5" + }, + "keywords": [ + "feature flags", + "wordpress", + "plugin" + ] } diff --git a/src/components/modals/__tests__/SdkModal.test.js b/src/components/modals/__tests__/SdkModal.test.js index ed66599..42dd786 100644 --- a/src/components/modals/__tests__/SdkModal.test.js +++ b/src/components/modals/__tests__/SdkModal.test.js @@ -4,7 +4,7 @@ import SdkModal from '../SdkModal'; describe('SdkModal component', () => { test('should render modal correctly', async () => { - const item = { name: 'Test Flag' }; + const item = { id: 1, name: 'Test Flag', enabled: false }; const closeSdkModal = jest.fn(); render(); diff --git a/tests/performance/cli/results.js b/tests/performance/cli/results.js new file mode 100644 index 0000000..b1bcd80 --- /dev/null +++ b/tests/performance/cli/results.js @@ -0,0 +1,253 @@ +#!/usr/bin/env node +/** + * External dependencies + */ +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { existsSync, readFileSync, writeFileSync } = require('node:fs'); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { join, resolve } = require('node:path'); + +process.env.WP_ARTIFACTS_PATH ??= join(process.cwd(), 'artifacts'); + +const args = process.argv.slice(2); + +const beforeFile = args[1]; +const afterFile = args[0] || resolve('./artifacts/performance-results.json'); + +if (!existsSync(afterFile)) { + // eslint-disable-next-line no-console + console.error(`File not found: ${afterFile}`); + process.exit(1); +} + +if (beforeFile && !existsSync(beforeFile)) { + // eslint-disable-next-line no-console + console.error(`File not found: ${beforeFile}`); + process.exit(1); +} + +/** + * @param {unknown} v + * @return {string} Formatted value. + */ +function formatTableValue(v) { + if (v === true || v === 'true') return '✅'; + if (!v || v === 'false') return ''; + return v?.toString() || String(v); +} + +/** + * Simple way to format an array of objects as a Markdown table. + * + * For example, this array: + * + * [ + * { + * foo: 123, + * bar: 456, + * baz: 'Yes', + * }, + * { + * foo: 777, + * bar: 999, + * baz: 'No', + * } + * ] + * + * Will result in the following table: + * + * | foo | bar | baz | + * |-----|-----|-----| + * | 123 | 456 | Yes | + * | 777 | 999 | No | + * + * @param {Array} rows Table rows. + * @return {string} Markdown table content. + */ +function formatAsMarkdownTable(rows) { + let result = ''; + const headers = Object.keys(rows[0]); + for (const header of headers) { + result += `| ${header} `; + } + result += '|\n'; + for (const header of headers) { + const dashes = '-'.repeat(header.length); + result += `| ${dashes} `; + } + result += '|\n'; + + for (const row of rows) { + for (const [key, value] of Object.entries(row)) { + result += `| ${formatTableValue(value).padStart(key.length, ' ')} `; + } + result += '|\n'; + } + + return result; +} + +/** + * Computes the median number from an array numbers. + * + * @param {number[]} array List of numbers. + * @return {number} Median. + */ +function median(array) { + const mid = Math.floor(array.length / 2); + const numbers = [...array].sort((a, b) => a - b); + const result = + array.length % 2 !== 0 + ? numbers[mid] + : (numbers[mid - 1] + numbers[mid]) / 2; + + return Number(result.toFixed(2)); +} + +/** + * @type {Array<{file: string, title: string, results: Record[]}>} + */ +let beforeStats = []; + +/** + * @type {Array<{file: string, title: string, results: Record[]}>} + */ +let afterStats; + +if (beforeFile) { + try { + beforeStats = JSON.parse( + readFileSync(beforeFile, { encoding: 'utf-8' }) + ); + } catch {} +} + +try { + afterStats = JSON.parse(readFileSync(afterFile, { encoding: 'utf-8' })); +} catch { + // eslint-disable-next-line no-console + console.error(`Could not read file: ${afterFile}`); + process.exit(1); +} + +let summaryMarkdown = `**Performance Test Results**\n\n`; + +if (process.env.GITHUB_SHA) { + summaryMarkdown += `Performance test results for ${process.env.GITHUB_SHA} are in 🛎️!\n\n`; +} else { + summaryMarkdown += `Performance test results are in 🛎️!\n\n`; +} + +if (beforeFile) { + summaryMarkdown += `Note: the numbers in parentheses show the difference to the previous (baseline) test run.\n\n`; +} + +// eslint-disable-next-line no-console +console.log('Performance Test Results\n'); + +if (beforeFile) { + // eslint-disable-next-line no-console + console.log( + 'Note: the numbers in parentheses show the difference to the previous (baseline) test run.\n' + ); +} + +const DELTA_VARIANCE = 0.5; +const PERCENTAGE_VARIANCE = 2; + +/** + * Format value and add unit. + * + * Turns bytes into MB (base 10). + * + * @todo Dynamic formatting based on definition in result.json. + * + * @param {number} value Value. + * @param {string} key Key. + * @return {string} Formatted value. + */ +function formatValue(value, key) { + if (key === 'CLS') { + return value.toFixed(2); + } + + if (key === 'wpDbQueries') { + return value.toFixed(0); + } + + if (key === 'wpMemoryUsage') { + return `${(value / Math.pow(10, 6)).toFixed(2)} MB`; + } + + return `${value.toFixed(2)} ms`; +} + +for (const { file, title, results } of afterStats) { + const prevStat = beforeStats.find((s) => s.file === file); + + /** + * @type {Array>} + */ + const diffResults = []; + + for (const i in results) { + const newResult = results[i]; + + /** + * @type {Record} + */ + const diffResult = { + Run: i, + }; + + for (const [key, values] of Object.entries(newResult)) { + // Only do comparison if the number of results is the same. + const prevValues = + prevStat?.results.length === results.length + ? prevStat?.results[i].key + : null; + + const value = median(values); + const prevValue = prevValues ? median(prevValues) : 0; + const delta = value - prevValue; + const percentage = (delta / value) * 100; + + // Skip if there is not a significant delta or none at all. + if ( + !prevValues || + !percentage || + Math.abs(percentage) <= PERCENTAGE_VARIANCE || + !delta || + Math.abs(delta) <= DELTA_VARIANCE + ) { + diffResult[key] = formatValue( + /** @type {number} */ (value), + key + ); + continue; + } + + const prefix = delta > 0 ? '+' : ''; + + diffResult[key] = `${formatValue( + value, + key + )} (${prefix}${formatValue(delta, key)} / ${prefix}${percentage}%)`; + } + + diffResults.push(diffResult); + } + + // eslint-disable-next-line no-console + console.log(title); + // eslint-disable-next-line no-console + console.table(diffResults); + + summaryMarkdown += `**${title}**\n\n`; + summaryMarkdown += `${formatAsMarkdownTable(diffResults)}\n`; +} + +writeFileSync( + join(process.env.WP_ARTIFACTS_PATH, '/performance-results.md'), + summaryMarkdown +); diff --git a/tests/performance/config/performance-reporter.ts b/tests/performance/config/performance-reporter.ts new file mode 100644 index 0000000..0ed03b4 --- /dev/null +++ b/tests/performance/config/performance-reporter.ts @@ -0,0 +1,113 @@ +/* eslint-disable no-console */ +import { join } from 'node:path'; +import { writeFileSync } from 'node:fs'; +import type { + FullConfig, + FullResult, + Reporter, + TestCase, + TestResult, +} from '@playwright/test/reporter'; +import { median } from '../utils'; + +process.env.WP_ARTIFACTS_PATH ??= join(process.cwd(), 'artifacts'); + +class PerformanceReporter implements Reporter { + private shard?: FullConfig['shard']; + + allResults: Record< + string, + { + title: string; + results: Record[]; + } + > = {}; + + onBegin(config: FullConfig) { + if (config.shard) { + this.shard = config.shard; + } + } + + /** + * Called after a test has been finished in the worker process. + * + * Used to add test results to the final summary of all tests. + * + * @param test + * @param result + */ + onTestEnd(test: TestCase, result: TestResult) { + const performanceResults = result.attachments.find( + (attachment) => attachment.name === 'results' + ); + + if (performanceResults?.body) { + this.allResults[test.location.file] ??= { + // 0 = empty, 1 = browser, 2 = file name, 3 = test suite name. + title: test.titlePath()[3], + results: [], + }; + this.allResults[test.location.file].results.push( + JSON.parse(performanceResults.body.toString('utf-8')) + ); + } + } + + /** + * Called after all tests have been run, or testing has been interrupted. + * + * Provides a quick summary and writes all raw numbers to a file + * for further processing, for example to compare with a previous run. + * + * @param result + */ + onEnd(result: FullResult) { + const summary = []; + + if (Object.keys(this.allResults).length > 0) { + if (this.shard) { + console.log( + `\nPerformance Test Results ${this.shard.current}/${this.shard.total}` + ); + } else { + console.log(`\nPerformance Test Results`); + } + console.log(`Status: ${result.status}`); + } + + for (const [file, { title, results }] of Object.entries( + this.allResults + )) { + console.log(`\n${title}\n`); + console.table( + results.map((r) => + Object.fromEntries( + Object.entries(r).map(([key, value]) => [ + key, + median(value), + ]) + ) + ) + ); + + summary.push({ + file, + title, + results, + }); + } + + if (!this.shard) { + writeFileSync( + join( + process.env.WP_ARTIFACTS_PATH as string, + 'performance-results.json' + ), + JSON.stringify(summary, null, 2) + ); + } + } +} + +export default PerformanceReporter; diff --git a/tests/performance/playwright.config.ts b/tests/performance/playwright.config.ts new file mode 100644 index 0000000..ae56a4a --- /dev/null +++ b/tests/performance/playwright.config.ts @@ -0,0 +1,40 @@ +/** + * External dependencies + */ +import { join } from 'node:path'; +import { defineConfig, devices } from '@playwright/test'; +import dotenv from 'dotenv'; + +dotenv.config(); + +process.env.WP_ARTIFACTS_PATH ??= join(process.cwd(), 'artifacts'); +const authStoragePath = process.env.WP_AUTH_STORAGE; +process.env.STORAGE_STATE_PATH ??= join( + process.env.WP_ARTIFACTS_PATH, + authStoragePath +); +process.env.TEST_ITERATIONS ??= '4'; + +const config = defineConfig({ + globalSetup: require.resolve('../e2e/global-setup.ts'), + reporter: process.env.CI + ? [['blob'], ['./config/performance-reporter.ts']] + : [['list'], ['./config/performance-reporter.ts']], + forbidOnly: !!process.env.CI, + workers: 1, + retries: 0, + repeatEach: 1, + timeout: parseInt(process.env.TIMEOUT || '', 10) || 600_000, // Defaults to 10 minutes. + reportSlowTests: null, + use: { + baseURL: process.env.WP_BASE_URL, + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); + +export default config; diff --git a/tests/performance/specs/loadtime.spec.ts b/tests/performance/specs/loadtime.spec.ts new file mode 100644 index 0000000..38cf219 --- /dev/null +++ b/tests/performance/specs/loadtime.spec.ts @@ -0,0 +1,48 @@ +import { expect, test } from '@wordpress/e2e-test-utils-playwright'; +import { camelCaseDashes } from '../utils'; + +const results: Record = {}; + +test.use({ storageState: process.env.WP_AUTH_STORAGE }); +test.describe('Feature flags Perf Tests', () => { + // After all results are processed, attach results for further processing. + // For easier handling, only one attachment per file. + test.afterAll(async ({}, testInfo) => { + await testInfo.attach('results', { + body: JSON.stringify(results, null, 2), + contentType: 'application/json', + }); + }); + + const iterations = Number(process.env.TEST_ITERATIONS); + for (let i = 1; i <= iterations; i++) { + test(`Measure load time metrics (${i} of ${iterations})`, async ({ + page, + admin, + metrics, + }) => { + await admin.visitAdminPage('/'); + await page.getByRole('link', { name: 'Feature Flags' }).click(); + await expect( + page.getByRole('heading', { name: 'Feature Flags' }) + ).toBeVisible(); + + const serverTiming = await metrics.getServerTiming(); + + for (const [key, value] of Object.entries(serverTiming)) { + results[camelCaseDashes(key)] ??= []; + results[camelCaseDashes(key)]?.push(value); + } + + const ttfb = await metrics.getTimeToFirstByte(); + const lcp = await metrics.getLargestContentfulPaint(); + + results.largestContentfulPaint ??= []; + results.largestContentfulPaint.push(lcp); + results.timeToFirstByte ??= []; + results.timeToFirstByte.push(ttfb); + results.lcpMinusTtfb ??= []; + results.lcpMinusTtfb.push(lcp - ttfb); + }); + } +}); diff --git a/tests/performance/utils/index.ts b/tests/performance/utils/index.ts new file mode 100644 index 0000000..8d67ae0 --- /dev/null +++ b/tests/performance/utils/index.ts @@ -0,0 +1,27 @@ +/** + * Helper function to camel case the letter after dashes, removing the dashes. + * + * @param str + */ +export function camelCaseDashes(str: string) { + return str.replace(/-([a-z])/g, function (g) { + return g[1].toUpperCase(); + }); +} + +/** + * Computes the median number from an array numbers. + * + * @param array List of numbers. + * @return Median. + */ +export function median(array: number[]) { + const mid = Math.floor(array.length / 2); + const numbers = [...array].sort((a, b) => a - b); + const result = + array.length % 2 !== 0 + ? numbers[mid] + : (numbers[mid - 1] + numbers[mid]) / 2; + + return Number(result.toFixed(2)); +}