diff --git a/docs/config/index.md b/docs/config/index.md index c4d6ed1ca9b5..d6127a7d294c 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -1481,7 +1481,26 @@ Do not show files with 100% statement, branch, and function coverage. #### coverage.thresholds -Options for coverage thresholds +Options for coverage thresholds. + +If a threshold is set to a positive number, it will be interpreted as the minimum percentage of coverage required. For example, setting the lines threshold to `90` means that 90% of lines must be covered. + +If a threshold is set to a negative number, it will be treated as the maximum number of uncovered items allowed. For example, setting the lines threshold to `-10` means that no more than 10 lines may be uncovered. + + +```ts +{ + coverage: { + thresholds: { + // Requires 90% function coverage + functions: 90, + + // Require that no more than 10 lines are uncovered + lines: -10, + } + } +} +``` ##### coverage.thresholds.lines @@ -1490,7 +1509,6 @@ Options for coverage thresholds - **CLI:** `--coverage.thresholds.lines=` Global threshold for lines. -See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. ##### coverage.thresholds.functions @@ -1499,7 +1517,6 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-threshol - **CLI:** `--coverage.thresholds.functions=` Global threshold for functions. -See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. ##### coverage.thresholds.branches @@ -1508,7 +1525,6 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-threshol - **CLI:** `--coverage.thresholds.branches=` Global threshold for branches. -See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. ##### coverage.thresholds.statements @@ -1517,7 +1533,6 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-threshol - **CLI:** `--coverage.thresholds.statements=` Global threshold for statements. -See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. ##### coverage.thresholds.perFile @@ -1535,7 +1550,7 @@ Check thresholds per file. - **Available for providers:** `'v8' | 'istanbul'` - **CLI:** `--coverage.thresholds.autoUpdate=` -Update all threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds. +Update all threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is better than the configured thresholds. This option helps to maintain thresholds when coverage is improved. ##### coverage.thresholds.100 diff --git a/packages/vitest/src/utils/coverage.ts b/packages/vitest/src/utils/coverage.ts index 2d60ef22cc63..3fba42411c01 100644 --- a/packages/vitest/src/utils/coverage.ts +++ b/packages/vitest/src/utils/coverage.ts @@ -363,25 +363,54 @@ export class BaseCoverageProvider= 0) { const coverage = summary.data[thresholdKey].pct if (coverage < threshold) { process.exitCode = 1 - /* + /** * Generate error message based on perFile flag: * - ERROR: Coverage for statements (33.33%) does not meet threshold (85%) for src/math.ts * - ERROR: Coverage for statements (50%) does not meet global threshold (85%) */ - let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet ${ - name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"` + let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet ${name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"` } threshold (${threshold}%)` if (this.options.thresholds?.perFile && file) { errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}` } + this.ctx.logger.error(errorMessage) + } + } + else { + const uncovered = summary.data[thresholdKey].total - summary.data[thresholdKey].covered + const absoluteThreshold = threshold * -1 + + if (uncovered > absoluteThreshold) { + process.exitCode = 1 + + /** + * Generate error message based on perFile flag: + * - ERROR: Uncovered statements (33) exceed threshold (30) for src/math.ts + * - ERROR: Uncovered statements (33) exceed global threshold (30) + */ + let errorMessage = `ERROR: Uncovered ${thresholdKey} (${uncovered}) exceed ${name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"` + } threshold (${absoluteThreshold})` + + if (this.options.thresholds?.perFile && file) { + errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}` + } + this.ctx.logger.error(errorMessage) } } @@ -416,12 +445,30 @@ export class BaseCoverageProvider summary[key].pct), - ) + /** + * Positive thresholds are treated as minimum coverage percentages (X means: X% of lines must be covered), + * while negative thresholds are treated as maximum uncovered counts (-X means: X lines may be uncovered). + */ + if (threshold >= 0) { + const actual = Math.min( + ...summaries.map(summary => summary[key].pct), + ) - if (actual > threshold) { - thresholdsToUpdate.push([key, actual]) + if (actual > threshold) { + thresholdsToUpdate.push([key, actual]) + } + } + else { + const absoluteThreshold = threshold * -1 + const actual = Math.max( + ...summaries.map(summary => summary[key].total - summary[key].covered), + ) + + if (actual < absoluteThreshold) { + // If everything was covered, set new threshold to 100% (since a threshold of 0 would be considered as 0%) + const updatedThreshold = actual === 0 ? 100 : actual * -1 + thresholdsToUpdate.push([key, updatedThreshold]) + } } } diff --git a/test/coverage-test/fixtures/configs/vitest.config.thresholds-auto-update.ts b/test/coverage-test/fixtures/configs/vitest.config.thresholds-auto-update.ts index ca57c233a41e..57dd4a621946 100644 --- a/test/coverage-test/fixtures/configs/vitest.config.thresholds-auto-update.ts +++ b/test/coverage-test/fixtures/configs/vitest.config.thresholds-auto-update.ts @@ -9,14 +9,14 @@ export default defineConfig({ // Global ones lines: 0.1, functions: 0.2, - branches: 0.3, - statements: 0.4, + branches: -1000, + statements: -2000, '**/src/math.ts': { branches: 0.1, functions: 0.2, - lines: 0.3, - statements: 0.4 + lines: -1000, + statements: -2000, } } } diff --git a/test/coverage-test/test/threshold-auto-update.test.ts b/test/coverage-test/test/threshold-auto-update.test.ts index f39ea10158b9..529b0c64087a 100644 --- a/test/coverage-test/test/threshold-auto-update.test.ts +++ b/test/coverage-test/test/threshold-auto-update.test.ts @@ -20,14 +20,14 @@ test('thresholds.autoUpdate updates thresholds', async () => { // Global ones lines: 0.1, functions: 0.2, - branches: 0.3, - statements: 0.4, + branches: -1000, + statements: -2000, '**/src/math.ts': { branches: 0.1, functions: 0.2, - lines: 0.3, - statements: 0.4 + lines: -1000, + statements: -2000, } } } @@ -56,13 +56,13 @@ test('thresholds.autoUpdate updates thresholds', async () => { lines: 55.55, functions: 33.33, branches: 100, - statements: 55.55, + statements: -8, '**/src/math.ts': { branches: 100, functions: 25, - lines: 50, - statements: 50 + lines: -6, + statements: -6, } } } @@ -84,13 +84,13 @@ test('thresholds.autoUpdate updates thresholds', async () => { lines: 33.33, functions: 33.33, branches: 100, - statements: 33.33, + statements: -4, '**/src/math.ts': { branches: 100, functions: 25, - lines: 25, - statements: 25 + lines: -3, + statements: -3, } } } diff --git a/test/coverage-test/test/threshold-failure.test.ts b/test/coverage-test/test/threshold-failure.test.ts index 6b461b1a24d3..0b66ed6201b9 100644 --- a/test/coverage-test/test/threshold-failure.test.ts +++ b/test/coverage-test/test/threshold-failure.test.ts @@ -2,7 +2,7 @@ import { expect } from 'vitest' import { sum } from '../fixtures/src/math' import { coverageTest, isV8Provider, normalizeURL, runVitest, test } from '../utils' -test('failing thresholds', async () => { +test('failing percentage thresholds', async () => { const { exitCode, stderr } = await runVitest({ include: [normalizeURL(import.meta.url)], coverage: { @@ -28,6 +28,36 @@ test('failing thresholds', async () => { expect(stderr).toContain('ERROR: Coverage for functions (25%) does not meet "**/fixtures/src/math.ts" threshold (100%)') }) +test('failing absolute thresholds', async () => { + const { exitCode, stderr } = await runVitest({ + include: [normalizeURL(import.meta.url)], + coverage: { + all: false, + include: ['**/fixtures/src/math.ts'], + thresholds: { + '**/fixtures/src/math.ts': { + branches: -1, + functions: -2, + lines: -5, + statements: -1, + }, + }, + }, + }, { throwOnError: false }) + + expect(exitCode).toBe(1) + + if (isV8Provider()) { + expect(stderr).toContain('ERROR: Uncovered lines (6) exceed "**/fixtures/src/math.ts" threshold (5)') + expect(stderr).toContain('ERROR: Uncovered functions (3) exceed "**/fixtures/src/math.ts" threshold (2)') + expect(stderr).toContain('ERROR: Uncovered statements (6) exceed "**/fixtures/src/math.ts" threshold (1)') + } + else { + expect(stderr).toContain('ERROR: Uncovered functions (3) exceed "**/fixtures/src/math.ts" threshold (2)') + expect(stderr).toContain('ERROR: Uncovered statements (3) exceed "**/fixtures/src/math.ts" threshold (1)') + } +}) + coverageTest('cover some lines, but not too much', () => { expect(sum(1, 2)).toBe(3) })