diff --git a/docs/testing.i18n.yaml b/docs/testing.i18n.yaml index b870988726..d7936c0fef 100644 --- a/docs/testing.i18n.yaml +++ b/docs/testing.i18n.yaml @@ -9,8 +9,8 @@ en: c024da7e8d9cb056 zh: c4cc1a66d2903656 /testing-policy/how-specs-execute: - en: 298c13b105e86eae - zh: 4f1b0b7742fb7b49 + en: 5d8eb65958da096d + zh: 4a40f1c5fe5b2d14 /testing-policy/the-with-key-policy-inference-is-cheap-here: en: d555af53068bf523 zh: ddc4265e5de5b3be diff --git a/docs/testing.md b/docs/testing.md index 21b2916a92..9ea6db1501 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -18,7 +18,7 @@ Session fixtures retain headers and payloads but omit body sequence/time envelop ## How specs execute -Forked workers run several spec files at once, the coverage gate splits into concurrent partitions beside the other gates in its job, and the self-hosted runners share one host and one volume. Only the process is isolated: ports, predictable paths, external namespaces, and inherited children are not. Own each acquired resource through its teardown, and read a spec that passes only when it runs alone as a defect in the spec rather than an unstable runner. [dsh-ci-test-reliability](../.agents/skills/dsh-ci-test-reliability/SKILL.md) owns the allocation, restoration, synchronization, timeout-budget, platform, and teardown rules; its [flake diagnosis workflow](../.agents/skills/dsh-ci-test-reliability/references/ci-flake-diagnosis.md) classifies an existing probabilistic failure. +Forked workers run several spec files at once, the coverage gate splits into concurrent partitions beside the other gates in its job, and the self-hosted runners share one host and one volume. Only the process is isolated: ports, predictable paths, external namespaces, and inherited children are not. Own each acquired resource through teardown. A spec passing only alone has a test defect, not an unstable runner. [dsh-ci-test-reliability](../.agents/skills/dsh-ci-test-reliability/SKILL.md) owns the allocation, restoration, synchronization, timeout-budget, platform, and teardown rules; its [flake diagnosis workflow](../.agents/skills/dsh-ci-test-reliability/references/ci-flake-diagnosis.md) classifies an existing probabilistic failure. [Unexpected fork exits](../scripts/coverage-fork-diagnostics.ts) report PID, file, code, and signal. ## The with-key policy: inference is cheap here diff --git a/docs/testing.zh.md b/docs/testing.zh.md index 714d5a8509..81c16c60ab 100644 --- a/docs/testing.zh.md +++ b/docs/testing.zh.md @@ -18,7 +18,7 @@ Session fixture 保留 header 与 payload,但省略正文 seq/time envelope; ## spec 如何被执行 -fork 出的 worker 会同时运行多个 spec 文件,coverage gate 会拆成并发的 partition,与同一个 job 中的其它 gate 并排运行,而自托管 runner 共用同一台宿主机和同一个卷。被隔离的只有进程:端口、可预测路径、外部命名空间和继承而来的子进程都不隔离。为每个占用的资源负责到它的 teardown,并把「只有单独运行时才通过」的 spec 读作该 spec 的缺陷,而不是 runner 不稳定。[dsh-ci-test-reliability](../.agents/skills/dsh-ci-test-reliability/SKILL.md) 负责资源分配、状态恢复、同步、超时预算、平台差异与 teardown 规则;它的 [flake 诊断流程](../.agents/skills/dsh-ci-test-reliability/references/ci-flake-diagnosis.md)用于归类已经存在的概率性失败。 +fork 出的 worker 会同时运行多个 spec 文件,coverage gate 会拆成并发的 partition,与同一个 job 中的其它 gate 并排运行,而自托管 runner 共用同一台宿主机和同一个卷。被隔离的只有进程:端口、可预测路径、外部命名空间和继承而来的子进程都不隔离。为每个占用的资源负责到 teardown。只在单独运行时通过的 spec 存在测试缺陷,而非 runner 不稳定。[dsh-ci-test-reliability](../.agents/skills/dsh-ci-test-reliability/SKILL.md) 负责资源分配、状态恢复、同步、超时预算、平台差异与 teardown 规则;它的 [flake 诊断流程](../.agents/skills/dsh-ci-test-reliability/references/ci-flake-diagnosis.md)用于归类已经存在的概率性失败。 [fork 意外退出](../scripts/coverage-fork-diagnostics.ts)时会报告 PID、文件、退出码和信号。 ## 带密钥策略:推理(inference)在这里很便宜 diff --git a/scripts/coverage-fork-diagnostics.spec.ts b/scripts/coverage-fork-diagnostics.spec.ts new file mode 100644 index 0000000000..a06eec13e6 --- /dev/null +++ b/scripts/coverage-fork-diagnostics.spec.ts @@ -0,0 +1,277 @@ +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { runGate } from './run-gates.ts' +import { PassThrough } from 'node:stream' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ForksPoolWorker } from 'vitest/node' +import type { PoolOptions, PoolWorker, TestProject, WorkerRequest, WorkerResponse } from 'vitest/node' +import { coverageForkPool } from './coverage-fork-diagnostics.ts' + +const roots: string[] = [] +const cleanups: Array<() => Promise> = [] + +afterEach(async () => { + const cleanup = cleanups.splice(0) + const directories = roots.splice(0) + const results = await Promise.allSettled(cleanup.map(dispose => dispose())) + vi.restoreAllMocks() + for (const directory of directories) await rm(directory, { recursive: true, force: true }) + const errors: unknown[] = [] + for (const result of results) { + if (result.status === 'rejected') errors.push(result.reason) + } + if (errors.length > 0) throw new AggregateError(errors, 'fork fixture cleanup failed') +}) + +async function fixture(exitCode: number, early = false): Promise { + const root = await mkdtemp(join(tmpdir(), 'dsh-coverage-fork-')) + roots.push(root) + await mkdir(join(root, 'workers')) + // The public pool's distPath selects a plain Node protocol fixture, not a nested Vitest run. + await writeFile(join(root, 'workers/forks.js'), early ? `process.exit(${exitCode})\n` : ` +process.on('message', (message) => { + if (message.type === 'run' || message.type === 'collect') process.send({ kind: 'ready', pid: process.pid }) + if (message.type === 'cancel') process.exit(${exitCode}) + if (message.type === 'stop') process.exit(0) +}) +`) + return root +} + +function poolOptions(root: string) { + const outputStream = new PassThrough() + const errorStream = new PassThrough() + const error = vi.fn<(message: string) => void>() + // ForksPoolWorker needs only the logger and project name from the Vitest project. + const logger: Pick = { + outputStream, errorStream, error, + } + const project = { name: 'fixture', vitest: { logger } } as TestProject + const options: PoolOptions = { + distPath: root, project, method: 'run', environment: { name: 'node', options: null }, + execArgv: [], env: {}, + } + return { options, error, outputStream, errorStream } +} + +async function start(root: string, diagnostic = true) { + const { options, error, outputStream, errorStream } = poolOptions(root) + const pool: PoolWorker = diagnostic ? coverageForkPool.createPoolWorker(options) : new ForksPoolWorker(options) + let started = false + const closed = Promise.withResolvers() + cleanups.push(async () => { + if (started) { + await pool.stop() + await closed.promise + } + outputStream.destroy() + errorStream.destroy() + }) + await pool.start() + started = true + pool.on('close', () => { closed.resolve(undefined) }) + const exited = new Promise((resolve) => { pool.on('exit', resolve) }) + const ready = new Promise((resolve) => { + pool.on('message', (message: unknown) => { + const response = message as { kind: string; pid: number } + if (response.kind === 'ready') resolve(response.pid) + }) + }) + return { pool, ready, exited, error } +} + +function request(filepath: string, workerId: number): Extract { + return { + __vitest_worker_request__: true, type: 'run', + context: { files: [{ filepath }], workerId, providedContext: {}, environment: { name: 'node', options: null } }, + } +} + +const cancel: WorkerRequest = { __vitest_worker_request__: true, type: 'cancel' } +const stop: WorkerRequest = { __vitest_worker_request__: true, type: 'stop' } + +async function configuredPool(pool: 'stock' | 'diagnostic', outcome: 'exit' | 'pass', signal: AbortSignal) { + const cacheDir = await mkdtemp(join(tmpdir(), 'dsh-configured-fork-cache-')) + roots.push(cacheDir) + const controller = new AbortController() + const result = runGate({ + id: `fork-${pool}`, label: 'fork fixture', displayCommand: 'Vitest pool fixture', + command: process.execPath, + args: [resolve('node_modules/vitest/vitest.mjs'), 'run', '--config', 'scripts/fixtures/coverage-fork.config.ts'], + env: { DSH_TEST_FORK_POOL: pool, DSH_TEST_FORK_OUTCOME: outcome, DSH_TEST_FORK_CACHE: cacheDir }, + }, AbortSignal.any([signal, controller.signal])) + let settled = false + cleanups.push(async () => { + if (!settled) controller.abort() + await result + }) + try { + return await result + } finally { + settled = true + } +} + +describe('coverage fork configuration entry', () => { + it('preserves the real Vitest failure and adds metadata before its error report', async ({ signal }) => { + const [stock, diagnostic] = await Promise.all([ + configuredPool('stock', 'exit', signal), + configuredPool('diagnostic', 'exit', signal), + ]) + for (const result of [stock, diagnostic]) { + expect(result.aborted).not.toBe(true) + expect(result.error).toBeUndefined() + expect(result.signalCode).toBeNull() + expect(result.exitCode).toBe(1) + expect(result.output.map(chunk => chunk.text).join('')).toContain('Worker exited unexpectedly') + } + const output = diagnostic.output.map(chunk => chunk.text).join('') + expect(stock.output.map(chunk => chunk.text).join('')).not.toContain('coverage-worker-exit:') + const encoded = /coverage-worker-exit: (\{[^\r\n]*\})/u.exec(output)?.[1] + expect(encoded).toBeDefined() + const record = JSON.parse(encoded!) as { + pid: number + project: string + workerId: number + files: string[] + exitCode: number + signalCode: null + } + const pid = Number(/coverage-fixture-pid: (\d+)/u.exec(output)?.[1]) + const anyNumber: unknown = expect.any(Number) + const anyString: unknown = expect.any(String) + expect(record).toEqual({ + pid, project: 'configured-fork', workerId: anyNumber, + files: [anyString], exitCode: 23, signalCode: null, + }) + expect(record.files.map(file => file.replaceAll('\\', '/'))) + .toEqual([resolve('scripts/fixtures/coverage-fork.fixture.ts').replaceAll('\\', '/')]) + expect(output.indexOf('coverage-worker-exit:')).toBeLessThan(output.indexOf('Worker exited unexpectedly')) + }) + + it('keeps a successful configured run silent and successful', async ({ signal }) => { + const result = await configuredPool('diagnostic', 'pass', signal) + expect(result.aborted).not.toBe(true) + expect(result.error).toBeUndefined() + expect(result.signalCode).toBeNull() + expect(result.exitCode).toBe(0) + expect(result.output.map(chunk => chunk.text).join('')).not.toContain('coverage-worker-exit:') + }) +}) + +describe('coverage fork exit callback', () => { + it.each([ + ['native receiver', { pid: 41 }, 41], + ['unavailable receiver', undefined, null], + ] as const)('retains a signal with %s and silences expected shutdown', async (_name, receiver, pid) => { + let exitListener: ((value: unknown) => void) | undefined + vi.spyOn(ForksPoolWorker.prototype, 'start').mockResolvedValue(undefined) + vi.spyOn(ForksPoolWorker.prototype, 'stop').mockResolvedValue(undefined) + vi.spyOn(ForksPoolWorker.prototype, 'send').mockImplementation(() => undefined) + vi.spyOn(ForksPoolWorker.prototype, 'off').mockImplementation(() => undefined) + vi.spyOn(ForksPoolWorker.prototype, 'on').mockImplementation((event, callback) => { + if (event === 'exit') exitListener = callback + }) + const { options, error, outputStream, errorStream } = poolOptions('/unused') + const pool = coverageForkPool.createPoolWorker(options) + cleanups.push(async () => { + await pool.stop() + outputStream.destroy() + errorStream.destroy() + }) + await pool.start() + pool.send(request('/signal.spec.ts', 9)) + if (exitListener === undefined) throw new Error('exit observer was not registered') + Reflect.apply(exitListener, receiver, [null, 'SIGTERM']) + expect(error).toHaveBeenCalledWith(`coverage-worker-exit: ${JSON.stringify({ + pid, project: 'fixture', workerId: 9, files: ['/signal.spec.ts'], exitCode: null, signalCode: 'SIGTERM', + })}`) + error.mockClear() + pool.send(stop) + expect(() => { Reflect.apply(exitListener!, undefined, [null, 'SIGTERM']) }).not.toThrow() + expect(error).not.toHaveBeenCalled() + }) +}) + +describe('coverage fork exit diagnostics', () => { + it('attributes a later collect request instead of an earlier run request', async () => { + const instance = await start(await fixture(23)) + instance.pool.send(request('/earlier.spec.ts', 4)) + const pid = await instance.ready + instance.pool.send({ ...request('/collected.spec.ts', 5), type: 'collect' }) + const unrelated = { type: 'testfileFinished' } + expect(instance.pool.deserialize(unrelated)).toBe(unrelated) + expect(instance.pool.deserialize(null)).toBeNull() + instance.pool.send(cancel) + expect(await instance.exited).toBe(23) + expect(instance.error).toHaveBeenCalledWith(`coverage-worker-exit: ${JSON.stringify({ + pid, project: 'fixture', workerId: 5, files: ['/collected.spec.ts'], exitCode: 23, signalCode: null, + })}`) + }) + + it('does not attribute a post-completion exit to a finished file', async () => { + const instance = await start(await fixture(23)) + instance.pool.send(request('/finished.spec.ts', 6)) + const pid = await instance.ready + const finished = { + __vitest_worker_response__: true, type: 'testfileFinished', + } satisfies Extract + expect(instance.pool.deserialize(finished)).toBe(finished) + instance.pool.send(cancel) + expect(await instance.exited).toBe(23) + expect(instance.error).toHaveBeenCalledWith(`coverage-worker-exit: ${JSON.stringify({ + pid, project: 'fixture', workerId: null, files: [], exitCode: 23, signalCode: null, + })}`) + }) + + it('records the exact owner while an independent ready fork remains alive', async () => { + const root = await fixture(23) + const [first, second] = await Promise.all([start(root), start(root)]) + first.pool.send(request('/first.spec.ts', 1)) + second.pool.send(request('/second.spec.ts', 2)) + const [firstPid, secondPid] = await Promise.all([first.ready, second.ready]) + expect(firstPid).not.toBe(secondPid) + first.pool.send(cancel) + expect(await first.exited).toBe(23) + expect(first.error.mock.calls).toEqual([[`coverage-worker-exit: ${JSON.stringify({ + pid: firstPid, project: 'fixture', workerId: 1, files: ['/first.spec.ts'], exitCode: 23, signalCode: null, + })}`]]) + expect(second.error).not.toHaveBeenCalled() + expect(() => process.kill(secondPid, 0)).not.toThrow() + second.pool.send(stop) + expect(await second.exited).toBe(0) + expect(second.error).not.toHaveBeenCalled() + }) + + it('retains an unexpected zero exit instead of interpreting it as suite success', async () => { + const instance = await start(await fixture(0)) + instance.pool.send(request('/unfinished.spec.ts', 3)) + const pid = await instance.ready + instance.pool.send(cancel) + expect(await instance.exited).toBe(0) + expect(instance.error).toHaveBeenCalledWith(`coverage-worker-exit: ${JSON.stringify({ + pid, project: 'fixture', workerId: 3, files: ['/unfinished.spec.ts'], exitCode: 0, signalCode: null, + })}`) + }) + + it('records an exit before any file request without inventing a file owner', async () => { + const instance = await start(await fixture(31, true)) + expect(await instance.exited).toBe(31) + expect(instance.error).toHaveBeenCalledOnce() + const record = JSON.parse(instance.error.mock.calls[0]![0].slice('coverage-worker-exit: '.length)) as Record + const anyNumber: unknown = expect.any(Number) + expect(record).toEqual({ + pid: anyNumber, project: 'fixture', workerId: null, files: [], exitCode: 31, signalCode: null, + }) + }) + + it('observes the same child failure without diagnostics under the stock pool negative control', async () => { + const instance = await start(await fixture(23), false) + instance.pool.send(request('/stock.spec.ts', 4)) + await instance.ready + instance.pool.send(cancel) + expect(await instance.exited).toBe(23) + expect(instance.error).not.toHaveBeenCalled() + }) +}) diff --git a/scripts/coverage-fork-diagnostics.ts b/scripts/coverage-fork-diagnostics.ts new file mode 100644 index 0000000000..50d056d1d5 --- /dev/null +++ b/scripts/coverage-fork-diagnostics.ts @@ -0,0 +1,69 @@ +/** Add exit metadata without changing Vitest's fork scheduling or failure handling. */ +import type { ChildProcess } from 'node:child_process' +import { ForksPoolWorker } from 'vitest/node' +import type { PoolOptions, PoolRunnerInitializer, WorkerRequest, WorkerResponse } from 'vitest/node' + +const FINISHED_RESPONSE_TYPE: WorkerResponse['type'] = 'testfileFinished' + +/** Attribute only outstanding requests; stop silences exits, while cancel does not. */ +class DiagnosticForkWorker extends ForksPoolWorker { + private expectedExit = false + private observing = false + private request: Extract | undefined + private readonly reportExit: (this: ChildProcess | undefined, code: number | null, signal?: NodeJS.Signals | null) => void + + constructor(options: PoolOptions) { + super(options) + const report = (child: ChildProcess | undefined, code: number | null, signal: NodeJS.Signals | null | undefined): void => { + if (this.expectedExit) return + options.project.vitest.logger.error(`coverage-worker-exit: ${JSON.stringify({ + pid: child?.pid ?? null, + project: options.project.name, + workerId: this.request?.context.workerId ?? null, + files: this.request?.context.files.map(file => file.filepath) ?? [], + exitCode: code, + signalCode: signal ?? null, + })}`) + } + // ForksPoolWorker.on forwards Node ChildProcess events and their receiver. + this.reportExit = function (code, signal) { report(this, code, signal) } + } + + override async start(): Promise { + await super.start() + this.on('exit', this.reportExit) + this.observing = true + } + + override send(message: WorkerRequest): void { + if (message.type === 'run' || message.type === 'collect') this.request = message + if (message.type === 'stop') this.expectedExit = true + super.send(message) + } + + /** Vitest routes worker responses here; completion covers the whole run/collect request. */ + override deserialize(data: unknown): unknown { + const response = super.deserialize(data) + if (typeof response === 'object' && response !== null + && '__vitest_worker_response__' in response && response.__vitest_worker_response__ === true + && 'type' in response && response.type === FINISHED_RESPONSE_TYPE) { + this.request = undefined + } + return response + } + + override async stop(): Promise { + this.expectedExit = true + if (this.observing) { + this.off('exit', this.reportExit) + this.observing = false + } + await super.stop() + } +} + +/** Fork pool with additive process-exit diagnostics for coverage partitions. */ +export const coverageForkPool: PoolRunnerInitializer = { + name: 'coverage-forks', + createPoolWorker: options => new DiagnosticForkWorker(options), +} diff --git a/scripts/coverage-partitions.spec.ts b/scripts/coverage-partitions.spec.ts index 853207bc77..99bf586f81 100644 --- a/scripts/coverage-partitions.spec.ts +++ b/scripts/coverage-partitions.spec.ts @@ -540,6 +540,8 @@ describe('coverage partition coordinator', () => { expect(allConfigs).toContain('c.spec.ts') for (const source of partitionConfigs) { expect(source).toContain("from '../../vitest.config.ts'") + expect(source).toContain("from '../../scripts/coverage-fork-diagnostics.ts'") + expect(source).toContain('pool: coverageForkPool') } const mergeCommand = commands[3] if (mergeCommand === undefined) throw new Error('coverage merge command was not observed') diff --git a/scripts/coverage-partitions.ts b/scripts/coverage-partitions.ts index aafef76a60..5489861c85 100644 --- a/scripts/coverage-partitions.ts +++ b/scripts/coverage-partitions.ts @@ -429,6 +429,7 @@ function partitionConfigSource( const processBound = JSON.stringify(files.filter(file => projectOf.get(file) === 'process-bound').map(file => file.split('\\').join('/'))) return [ "import base from '../../vitest.config.ts'", + "import { coverageForkPool } from '../../scripts/coverage-fork-diagnostics.ts'", 'export default {', ' ...base,', ' test: {', @@ -437,6 +438,7 @@ function partitionConfigSource( ' ...project,', ' test: {', ' ...project.test,', + ' pool: coverageForkPool,', ' include: project.test.name === \'process-bound\' ? ' + processBound + ' : ' + threadSafe + ',', ' },', ' })),', diff --git a/scripts/fixtures/coverage-fork-exit.cjs b/scripts/fixtures/coverage-fork-exit.cjs new file mode 100644 index 0000000000..526ed84ef8 --- /dev/null +++ b/scripts/fixtures/coverage-fork-exit.cjs @@ -0,0 +1,2 @@ +// Preserve Node's exit before Vitest installs its test-level process.exit guard. +globalThis.__dshCoverageFixtureExit = process.exit.bind(process) diff --git a/scripts/fixtures/coverage-fork.config.ts b/scripts/fixtures/coverage-fork.config.ts new file mode 100644 index 0000000000..fc8df9f197 --- /dev/null +++ b/scripts/fixtures/coverage-fork.config.ts @@ -0,0 +1,18 @@ +import { resolve } from 'node:path' +import { defineConfig } from 'vitest/config' +import { coverageForkPool } from '../coverage-fork-diagnostics.ts' + +const cacheDir = process.env.DSH_TEST_FORK_CACHE +if (cacheDir === undefined || cacheDir === '') throw new Error('fork fixture requires its private cache directory') + +export default defineConfig({ + cacheDir, + test: { + name: 'configured-fork', + include: ['scripts/fixtures/coverage-fork.fixture.ts'], + pool: process.env.DSH_TEST_FORK_POOL === 'stock' ? 'forks' : coverageForkPool, + execArgv: ['--require', resolve(import.meta.dirname, 'coverage-fork-exit.cjs')], + maxWorkers: 1, + cache: false, + }, +}) diff --git a/scripts/fixtures/coverage-fork.fixture.ts b/scripts/fixtures/coverage-fork.fixture.ts new file mode 100644 index 0000000000..4ca9f5e6a6 --- /dev/null +++ b/scripts/fixtures/coverage-fork.fixture.ts @@ -0,0 +1,15 @@ +import { writeSync } from 'node:fs' +import { expect, it } from 'vitest' + +declare global { + /** Native process exit captured by this fixture's Node preload. */ + function __dshCoverageFixtureExit(code: number): never +} + +it('runs in the configured pool', () => { + writeSync(2, `coverage-fixture-pid: ${process.pid}\n`) + if (process.env.DSH_TEST_FORK_OUTCOME === 'exit') { + globalThis.__dshCoverageFixtureExit(23) + } + expect(2 + 2).toBe(4) +})