Files
deepseek-harness/packages/session/session-persistence-jsonl/src/migration-verifier.ts
T
Tianyi Cui 1103b0277b fix(session): await natural migration verifier worker exit
A successful verifier message was treated as terminal even though the
Worker Thread was still alive. The parent immediately terminated the
worker and detached cancellation, so an error, nonzero exit, or caller
cancellation after that message could be ignored and verification could
resolve successfully.

Keep the first successful response pending until an observed natural exit
with code 0. A missing response or nonzero exit rejects after the worker
has already exited. Error and abort callbacks retain the existing
terminate-and-join path, including its first-error and AggregateError
behavior. The scheduler stays unchanged: its permit remains held until
the verification promise settles, now after successful worker quiescence.
The worker already closes its parent port in finally, so success needs
neither a forced stop nor another protocol message.

Replace success-termination expectations with deterministic message/exit
barriers and cover post-response errors, nonzero exit, cancellation, and
permit handoff. Remove the obsolete hand-written termination thenable and
its retired unknown-assertion baseline entry. Add real source workers for
plain and Zstandard input, and require the built plain-Node migration smoke
to complete with zero termination calls. Document the settlement condition.

Native Windows CI recorded process access violation 0xC0000005 while a
migration fixture was active. Removing unnecessary successful-worker
termination is consistent with that evidence, but this commit does not
claim that the native faulting module or mechanism has been established.
The deterministic regressions prove the lifecycle defect independently;
the next native CI run must assess whether the observed crash recurs.

Validation:
- Original source failed all new message-to-exit regression scenarios;
  the updated focused lifecycle file passes 19 tests with 100% coverage.
- Node 24 owning package: 19 files, 741 passed, 1 skipped; every included
  package source file reaches 100% statements/branches/functions/lines.
- Real source workers passed for none/zstd; built plain-Node smoke passed
  with zero terminate calls and no timeout or process signal.
- Exact Node 22.19 source-mode none/zstd workers naturally exited 0 with
  zero terminate calls, exercising the async Module.register fallback.
- Host TypeScript build, focused lint, no-new-unknown-assertion gate,
  README translation pairing, and all 42 doc-sync gates passed.

Related to #4387 and #4394.
2026-10-06 11:29:17 +08:00

199 lines
6.6 KiB
TypeScript

/** Isolated verification for a staged or competing current JSONL generation. */
import { Worker } from 'node:worker_threads'
import type { WorkerOptions } from 'node:worker_threads'
import type { JsonlCompression } from './format.ts'
import type { JsonlExpectedPrefix, JsonlVerifiedGeneration } from './generation.ts'
interface VerificationRequest {
readonly path: string
readonly compression: JsonlCompression
readonly expectedId: string
readonly expectedEventCount: number
readonly expectedPrefix?: JsonlExpectedPrefix
}
type VerificationResponse =
| { readonly ok: true; readonly result: JsonlVerifiedGeneration }
| { readonly ok: false; readonly message: string; readonly stack?: string }
/** Process-wide memory bound for full-generation verification isolates. */
const MAX_CONCURRENT_VERIFIERS = 2
class VerificationScheduler {
private active = 0
private readonly waiting: Array<{ grant(): void }> = []
async run<T>(operation: () => Promise<T>, signal?: AbortSignal): Promise<T> {
const permit = this.acquire(signal)
if (permit !== undefined) await permit
try {
signal?.throwIfAborted()
return await operation()
} finally {
this.release()
}
}
private acquire(signal?: AbortSignal): Promise<void> | undefined {
signal?.throwIfAborted()
if (this.active < MAX_CONCURRENT_VERIFIERS) {
this.active += 1
return
}
return new Promise<void>((resolve, reject) => {
const waiter = {
grant: (): void => {
signal?.removeEventListener('abort', abort)
resolve()
},
}
const abort = (): void => {
const index = this.waiting.indexOf(waiter)
this.waiting.splice(index, 1)
reject(verifierAbortError(signal))
}
this.waiting.push(waiter)
signal?.addEventListener('abort', abort, { once: true })
})
}
private release(): void {
const next = this.waiting.shift()
if (next === undefined) {
this.active -= 1
return
}
next.grant()
}
}
const verificationScheduler = new VerificationScheduler()
function workerSpawn(request: VerificationRequest): { readonly entry: string | URL; readonly options: WorkerOptions } {
/* v8 ignore next 3 -- built-worker coverage owns the bundled path. */
if (!import.meta.url.endsWith('.ts')) {
return {
entry: new URL('./worker.cjs', import.meta.url),
options: { workerData: request, execArgv: [] },
}
}
const workerEntry = new URL('./worker.ts', import.meta.url)
const bootstrap = [
`import { register as registerEsm } from ${JSON.stringify(import.meta.resolve('tsx/esm/api'))}`,
`import { register as registerCjs } from ${JSON.stringify(import.meta.resolve('tsx/cjs/api'))}`,
'registerCjs()',
'registerEsm()',
`await import(${JSON.stringify(workerEntry.href)})`,
].join('\n')
return {
entry: new URL(`data:text/javascript,${encodeURIComponent(bootstrap)}`),
options: {
workerData: request,
execArgv: [],
},
}
}
/**
* Verify one current generation in a fresh Worker Thread.
* @param path - staged or competing current-generation path.
* @param compression - configured physical encoding.
* @param expectedId - Session id expected in the decoded header.
* @param expectedEventCount - exact logical event count expected after decoding.
* @param expectedPrefix - verified physical prefix; an append tail may be present and is not validated.
* @param signal - optional cancellation for scheduler wait and Worker execution.
* @returns the worker's physical identity and digest after its natural zero exit.
*/
export function verifyCurrentGenerationInWorker(
path: string,
compression: JsonlCompression,
expectedId: string,
expectedEventCount: number,
expectedPrefix?: JsonlExpectedPrefix,
signal?: AbortSignal,
): Promise<JsonlVerifiedGeneration> {
return verificationScheduler.run(() => runVerificationWorker(
path,
compression,
expectedId,
expectedEventCount,
expectedPrefix,
signal,
), signal)
}
function runVerificationWorker(
path: string,
compression: JsonlCompression,
expectedId: string,
expectedEventCount: number,
expectedPrefix?: JsonlExpectedPrefix,
signal?: AbortSignal,
): Promise<JsonlVerifiedGeneration> {
signal?.throwIfAborted()
const request: VerificationRequest = {
path, compression, expectedId, expectedEventCount,
...(expectedPrefix === undefined ? {} : { expectedPrefix }),
}
const { entry, options } = workerSpawn(request)
const worker = new Worker(entry, options)
return new Promise((resolve, reject) => {
let settled = false
let success: Extract<VerificationResponse, { ok: true }> | undefined
const cleanup = (): void => {
signal?.removeEventListener('abort', abort)
}
const fail = (error: Error): void => {
/* v8 ignore next -- a late error/exit races only after another terminal callback settled. */
if (settled) return
settled = true
cleanup()
void worker.terminate().then(
() => { reject(error) },
(cleanup: unknown) => {
reject(new AggregateError([error, cleanup], 'migration verifier termination failed'))
},
)
}
worker.once('message', (value: unknown) => {
/* v8 ignore next -- a duplicate message races only after another terminal callback settled. */
if (settled) return
if (typeof value !== 'object' || value === null || typeof (value as { ok?: unknown }).ok !== 'boolean') {
fail(new Error('migration verifier returned an invalid response'))
return
}
const response = value as VerificationResponse
if (!response.ok) {
const error = new Error(response.message)
if (response.stack !== undefined) error.stack = response.stack
fail(error)
return
}
success = response
})
worker.once('error', fail)
worker.once('exit', (code) => {
if (settled) return
settled = true
cleanup()
if (success === undefined) {
reject(new Error(`migration verifier exited before reporting a result (code ${code})`))
} else if (code !== 0) {
reject(new Error(`migration verifier exited after reporting a result (code ${code})`))
} else {
resolve(success.result)
}
})
const abort = (): void => { fail(verifierAbortError(signal)) }
signal?.addEventListener('abort', abort, { once: true })
})
}
function verifierAbortError(signal?: AbortSignal): Error {
const reason: unknown = signal?.reason
return reason instanceof Error
? reason
: new Error('migration verifier aborted', { cause: reason })
}