From b33fee44a55cf0bb2f416a0e25564fb0ff9660ff Mon Sep 17 00:00:00 2001 From: saraeloop Date: Tue, 14 Apr 2026 09:52:25 -0700 Subject: [PATCH] test(e2e): add hermetic CLI E2E and unify test structure - add built-artifact E2E test under test/e2e/cli.e2e.ts - use deterministic local package-lock fixture and isolated temp workspace - assert stable JSON contract markers and .depgraph persistence - avoid brittle score assertions and live registry dependency - add test:e2e, pack:dry-run, and release:check scripts - wire release:check into prepublishOnly - update CI to run: install -> test -> build -> test:e2e - update publish workflow to include E2E and pack dry-run - unify test structure under test/ while keeping E2E explicitly separated via *.e2e.ts pattern verification: - pnpm test - pnpm run build - pnpm run test:e2e - pnpm run release:check --- .github/workflows/ci.yml | 3 + .github/workflows/publish.yml | 10 ++- .gitignore | 4 +- CONTRIBUTING.md | 13 ++- package.json | 7 +- test/e2e/cli.e2e.ts | 154 ++++++++++++++++++++++++++++++++++ 6 files changed, 184 insertions(+), 7 deletions(-) create mode 100644 test/e2e/cli.e2e.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afd8eff..bb4cd31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,3 +41,6 @@ jobs: - name: Build run: pnpm run build + + - name: End-to-end test + run: pnpm run test:e2e diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 69002ea..74111b7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,11 +33,17 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Test + run: pnpm test + - name: Build run: pnpm run build - - name: Test - run: pnpm test + - name: End-to-end test + run: pnpm run test:e2e + + - name: Verify package contents + run: pnpm run pack:dry-run - name: Publish to npm run: npm publish diff --git a/.gitignore b/.gitignore index 1b2fb6f..bc44a71 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,9 @@ build/ out/ *.tsbuildinfo -# npm pack (local tarballs; do not commit) +# npm pack / test artifacts (local; do not commit) *.tgz +.npm-pack-cache/ # Environment / secrets .env @@ -50,4 +51,3 @@ AGENTS.md # Internal local docs / ADRs .internal/ - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 254adec..b6e70d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,7 @@ pnpm install pnpm run lint pnpm test pnpm run build +pnpm run test:e2e ``` Run the CLI locally: @@ -41,7 +42,17 @@ with your benchmark cases. See internal documentation for the schema. - open an issue first for larger feature changes - keep changes scoped and explain the behavior change clearly - add or update tests when behavior changes -- run `pnpm run lint`, `pnpm test`, and `pnpm run build` before submitting +- run `pnpm run lint`, `pnpm test`, `pnpm run build`, and `pnpm run test:e2e` before submitting + +## Release Readiness + +Use the built-artifact verification path before cutting or publishing a release: + +```bash +pnpm run release:check +``` + +This verifies the source test suite, the compiled CLI entrypoint at `dist/cli/index.js`, and the package contents that would be published. ## Commit Quality diff --git a/package.json b/package.json index 3ff6277..054122f 100644 --- a/package.json +++ b/package.json @@ -42,10 +42,13 @@ "build": "tsc -p tsconfig.json", "benchmark": "node dist/benchmark/index.js", "dev": "tsx src/cli/index.ts", - "test": "node --import tsx --test", + "pack:dry-run": "npm --cache .npm-pack-cache pack --dry-run", + "release:check": "pnpm test && pnpm run build && pnpm run test:e2e && pnpm run pack:dry-run", + "test": "node --import tsx --test test/**/*.test.ts", + "test:e2e": "node --import tsx --test test/**/*.e2e.ts", "test:smoke": "node --import tsx --test test/json-renderer.test.ts test/review-renderer.test.ts test/evaluation-renderer.test.ts test/integrated-scenarios.test.ts", "lint": "tsc --noEmit", - "prepublishOnly": "pnpm run build" + "prepublishOnly": "pnpm run release:check" }, "packageManager": "pnpm@10.33.0", "dependencies": { diff --git a/test/e2e/cli.e2e.ts b/test/e2e/cli.e2e.ts new file mode 100644 index 0000000..af8446d --- /dev/null +++ b/test/e2e/cli.e2e.ts @@ -0,0 +1,154 @@ +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { access, mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { join, resolve } from 'node:path' +import { tmpdir } from 'node:os' +import test from 'node:test' + +import type { ScanResult } from '../src/domain/entities.js' + +const CLI_ENTRYPOINT = resolve(process.cwd(), 'dist/cli/index.js') + +test('built CLI completes a package-lock project scan and degrades unresolved dependency metadata honestly', async () => { + await assertBuiltCliExists() + + const projectRoot = await mkdtemp(join(tmpdir(), 'depgraph-e2e-')) + const packageLockPath = join(projectRoot, 'package-lock.json') + + await writeFile( + packageLockPath, + JSON.stringify({ + name: 'depgraph-e2e-fixture', + version: '1.0.0', + lockfileVersion: 3, + packages: { + '': { + name: 'depgraph-e2e-fixture', + version: '1.0.0', + dependencies: { + '@depgraph/e2e-hermetic-alpha-9d3f0a5c': '^1.0.0', + }, + }, + 'node_modules/@depgraph/e2e-hermetic-alpha-9d3f0a5c': { + version: '1.0.0', + resolved: 'https://registry.example/@depgraph/e2e-hermetic-alpha-9d3f0a5c/-/alpha-1.0.0.tgz', + integrity: 'sha512-alpha', + dependencies: { + '@depgraph/e2e-hermetic-beta-9d3f0a5c': '^1.0.0', + }, + }, + 'node_modules/@depgraph/e2e-hermetic-beta-9d3f0a5c': { + version: '1.0.0', + resolved: 'https://registry.example/@depgraph/e2e-hermetic-beta-9d3f0a5c/-/beta-1.0.0.tgz', + integrity: 'sha512-beta', + }, + }, + }), + 'utf8', + ) + + const command = await runCli( + ['scan', '--package-lock', packageLockPath, '--json', '--depth', '2'], + projectRoot, + ) + + assert.equal(command.exitCode, 0, command.stderr || 'expected successful scan exit code') + assert.equal(command.stderr.trim(), '') + + const result = JSON.parse(command.stdout) as ScanResult + + assert.equal(result.scan_mode, 'package_lock') + assert.equal(result.scan_target, 'depgraph-e2e-fixture') + assert.equal(result.baseline_record_id, null) + assert.equal(result.requested_depth, 2) + assert.match(result.record_id, /depgraph-e2e-fixture@1\.0\.0:depth=2$/) + assert.equal(result.root.key, 'depgraph-e2e-fixture@1.0.0') + assert.equal(result.root.is_project_root, true) + assert.equal(result.root.metadata_status, 'synthetic_project_root') + assert.equal(result.root.metadata_warning, null) + assert.equal(result.total_scanned, 3) + assert.deepEqual(result.edge_findings, []) + assert.equal(result.findings.length, 0) + assert.equal(result.suspicious_count, 0) + assert.equal(result.overall_risk_level, 'safe') + assert.equal(result.root.dependencies.length, 1) + assert.equal(result.root.dependencies[0]?.key, '@depgraph/e2e-hermetic-alpha-9d3f0a5c@1.0.0') + assert.equal(result.root.dependencies[0]?.metadata_status, 'unresolved_registry_lookup') + assert.equal( + result.root.dependencies[0]?.lockfile_resolved_url, + 'https://registry.example/@depgraph/e2e-hermetic-alpha-9d3f0a5c/-/alpha-1.0.0.tgz', + ) + assert.equal(result.root.dependencies[0]?.lockfile_integrity, 'sha512-alpha') + assert.equal( + result.root.dependencies[0]?.dependencies[0]?.key, + '@depgraph/e2e-hermetic-beta-9d3f0a5c@1.0.0', + ) + assert.equal( + result.root.dependencies[0]?.dependencies[0]?.metadata_status, + 'unresolved_registry_lookup', + ) + assert.equal( + result.root.dependencies[0]?.dependencies[0]?.lockfile_resolved_url, + 'https://registry.example/@depgraph/e2e-hermetic-beta-9d3f0a5c/-/beta-1.0.0.tgz', + ) + assert.ok( + result.warnings.some((warning) => warning.kind === 'unresolved_registry_lookup'), + 'expected unresolved registry lookup warning(s) for hermetic fixture packages', + ) + + const scanHistoryPath = join(projectRoot, '.depgraph', 'scans.jsonl') + const persistedHistory = await readFile(scanHistoryPath, 'utf8') + const persistedLines = persistedHistory.trim().split('\n') + const persistedRecord = JSON.parse(persistedLines[0] ?? '') as ScanResult & { scan_mode: string } + + assert.equal(persistedLines.length, 1) + assert.equal(persistedRecord.scan_mode, 'package_lock') + assert.equal(persistedRecord.scan_target, 'depgraph-e2e-fixture') +}) + +async function assertBuiltCliExists(): Promise { + try { + await access(CLI_ENTRYPOINT) + } catch { + assert.fail(`Built CLI not found at ${CLI_ENTRYPOINT}. Run "pnpm run build" before "pnpm run test:e2e".`) + } +} + +async function runCli( + args: string[], + cwd: string, +): Promise<{ exitCode: number | null; stdout: string; stderr: string }> { + return new Promise((resolvePromise, rejectPromise) => { + const child = spawn(process.execPath, [CLI_ENTRYPOINT, ...args], { + cwd, + env: { + ...process.env, + CI: 'true', + FORCE_COLOR: '0', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }) + let stdout = '' + let stderr = '' + + child.stdout.on('data', (chunk: Buffer | string) => { + stdout += chunk.toString() + }) + + child.stderr.on('data', (chunk: Buffer | string) => { + stderr += chunk.toString() + }) + + child.on('error', (error) => { + rejectPromise(error) + }) + + child.on('close', (exitCode) => { + resolvePromise({ + exitCode, + stdout: stdout.trim(), + stderr: stderr.trim(), + }) + }) + }) +}