Merge pull request #5716 from deepseek-harness/worktree/pr-ci-speed-independent-20261005
perf(ci): reduce repeated compilation without raising concurrency
This commit is contained in:
commit
8a7d5014bf
39 files changed
+1584
-130
No files matched your search
@@ -23,6 +23,78 @@ function run(env: NodeJS.ProcessEnv, ...args: string[]) {
|
||||
}
|
||||
|
||||
describe('Python runtime executable builder CLI', () => {
|
||||
it.each([
|
||||
{ flags: [], command: 'run build' },
|
||||
{ flags: ['--artifacts-only'], command: 'run build --artifacts-only' },
|
||||
])('validates the closure before $command and deploys only after building', ({ flags, command }) => {
|
||||
const result = run(
|
||||
{ npm_execpath: 'C:\\tools\\pnpm.cjs' },
|
||||
'--dry-run',
|
||||
'--targets=node24-macos-arm64',
|
||||
...flags,
|
||||
)
|
||||
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.signal).toBeNull()
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
const commands = result.stdout.split('\n').filter(line => line.includes('C:\\tools\\pnpm.cjs '))
|
||||
expect(commands[0]).toMatch(/ run verify-runtime-closure$/)
|
||||
expect(commands[1]?.endsWith(' ' + command)).toBe(true)
|
||||
expect(commands[2]).toContain(' --filter dsh-python-runtime-closure deploy ')
|
||||
})
|
||||
|
||||
it('rejects simultaneous build modes before invoking pnpm', () => {
|
||||
const setup = mkdtempSync(join(tmpdir(), 'dsh-python-build-mode-'))
|
||||
temporaryDirectories.push(setup)
|
||||
const entrypoint = join(setup, 'pnpm.cjs')
|
||||
const commands = join(setup, 'commands.jsonl')
|
||||
writeFileSync(entrypoint, [
|
||||
"require('node:fs').appendFileSync(process.env.DSH_PYTHON_BUILD_COMMANDS, JSON.stringify(process.argv.slice(2)) + '\\n')",
|
||||
'process.exit(31)',
|
||||
'',
|
||||
].join('\n'))
|
||||
|
||||
const result = run(
|
||||
{ npm_execpath: entrypoint, DSH_PYTHON_BUILD_COMMANDS: commands },
|
||||
'--skip-build',
|
||||
'--artifacts-only',
|
||||
'--targets=node24-macos-arm64',
|
||||
)
|
||||
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.signal).toBeNull()
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('--skip-build and --artifacts-only cannot be combined')
|
||||
expect(result.stdout).toBe('')
|
||||
expect(existsSync(commands)).toBe(false)
|
||||
})
|
||||
|
||||
it('stops before artifact building or deployment when the closure check fails', () => {
|
||||
const setup = mkdtempSync(join(tmpdir(), 'dsh-python-closure-'))
|
||||
temporaryDirectories.push(setup)
|
||||
const entrypoint = join(setup, 'pnpm.cjs')
|
||||
const commands = join(setup, 'commands.jsonl')
|
||||
writeFileSync(entrypoint, [
|
||||
"require('node:fs').appendFileSync(process.env.DSH_PYTHON_BUILD_COMMANDS, JSON.stringify(process.argv.slice(2)) + '\\n')",
|
||||
"if (process.argv.slice(2).join(' ') === 'run verify-runtime-closure') process.exit(31)",
|
||||
'process.exit(32)',
|
||||
'',
|
||||
].join('\n'))
|
||||
|
||||
const result = run(
|
||||
{ npm_execpath: entrypoint, DSH_PYTHON_BUILD_COMMANDS: commands },
|
||||
'--artifacts-only',
|
||||
'--targets=node24-macos-arm64',
|
||||
)
|
||||
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.signal).toBeNull()
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('runtime dependency closure failed (exit code 31)')
|
||||
expect(readFileSync(commands, 'utf8').trim().split('\n'))
|
||||
.toEqual([JSON.stringify(['run', 'verify-runtime-closure'])])
|
||||
})
|
||||
|
||||
it('keeps the single-file dispatcher on the Python packaging surface', () => {
|
||||
const bootstrapPath = resolve(root, 'python/sdk-runtime/runtime-bootstrap.mjs')
|
||||
const bootstrap = readFileSync(bootstrapPath, 'utf8')
|
||||
@@ -59,6 +131,7 @@ describe('Python runtime executable builder CLI', () => {
|
||||
|
||||
expect(result.status).toBe(0)
|
||||
expect(result.stdout).toContain(`${process.execPath} C:\\tools\\pnpm.cjs run verify-runtime-closure`)
|
||||
expect(result.stdout).not.toMatch(/\[dry-run\] .* run build(?: --artifacts-only)?$/m)
|
||||
expect(result.stdout).toContain(`${process.execPath} C:\\tools\\pnpm.cjs --filter dsh-python-runtime-closure deploy`)
|
||||
const deploy = result.stdout.split('\n').find(line => line.includes(' --filter dsh-python-runtime-closure deploy'))
|
||||
expect(deploy).toContain('--prod --config.allow-unused-patches=true')
|
||||
|
||||
@@ -156,8 +156,10 @@ class BuildCli {
|
||||
private constructor(
|
||||
/** Build targets; defaults to the host platform only. */
|
||||
readonly targets: readonly Target[],
|
||||
/** Skip step 1 (`pnpm run build`); lib/ artifacts must already exist. */
|
||||
/** Skip the package build; lib/ artifacts must already exist. */
|
||||
readonly skipBuild: boolean,
|
||||
/** Emit package and Web artifacts without repository test and script typechecks. */
|
||||
readonly artifactsOnly: boolean,
|
||||
/** Print every command and config patch instead of executing. */
|
||||
readonly dryRun: boolean,
|
||||
) {}
|
||||
@@ -181,6 +183,9 @@ class BuildCli {
|
||||
console.log(BuildCli.usage())
|
||||
process.exit(0)
|
||||
}
|
||||
if (values['skip-build'] && values['artifacts-only']) {
|
||||
throw new Error('build-exe-for-python-sdk: --skip-build and --artifacts-only cannot be combined.')
|
||||
}
|
||||
const targets = values.targets === undefined
|
||||
? [Target.host()]
|
||||
: values.targets.split(',').map(part => part.trim()).filter(part => part !== '').map(spec => Target.parse(spec))
|
||||
@@ -193,7 +198,7 @@ class BuildCli {
|
||||
}
|
||||
seen.add(key)
|
||||
}
|
||||
return new BuildCli(targets, values['skip-build'], values['dry-run'])
|
||||
return new BuildCli(targets, values['skip-build'], values['artifacts-only'], values['dry-run'])
|
||||
}
|
||||
|
||||
private static parseRaw(argv: string[]) {
|
||||
@@ -202,6 +207,7 @@ class BuildCli {
|
||||
options: {
|
||||
'targets': { type: 'string' },
|
||||
'skip-build': { type: 'boolean', default: false },
|
||||
'artifacts-only': { type: 'boolean', default: false },
|
||||
'dry-run': { type: 'boolean', default: false },
|
||||
'help': { type: 'boolean', default: false },
|
||||
},
|
||||
@@ -215,6 +221,7 @@ class BuildCli {
|
||||
' --targets=<t1,t2,...> pkg targets, e.g. node24-linux-x64,node24-linux-arm64,node24-macos-arm64,node24-macos-x64,node24-win-x64.',
|
||||
' Default: the host platform only (on node24).',
|
||||
' --skip-build skip `pnpm run build` (lib/ artifacts must already exist).',
|
||||
' --artifacts-only omit repository test and script typechecks from the artifact build.',
|
||||
' --dry-run print every command and config patch without executing.',
|
||||
' --help print this help.',
|
||||
'',
|
||||
@@ -281,7 +288,8 @@ class SingleExeBuild {
|
||||
console.log('build-exe-for-python-sdk: skipping pnpm run build (--skip-build)')
|
||||
return
|
||||
}
|
||||
await this.runPnpm('build', ['run', 'build'])
|
||||
const args = this.cli.artifactsOnly ? ['run', 'build', '--artifacts-only'] : ['run', 'build']
|
||||
await this.runPnpm('build', args)
|
||||
}
|
||||
|
||||
/** Clear and deploy the runtime closure into the node carrier. */
|
||||
|
||||
@@ -15,4 +15,14 @@ describe('repository build CLI', () => {
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('build: Node.js TypeScript type stripping is unavailable')
|
||||
})
|
||||
|
||||
it('rejects a value for the artifacts-only flag before any build step', () => {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
['--import', 'tsx/esm', resolve(root, 'scripts/build.ts'), '--artifacts-only=yes'],
|
||||
{ cwd: root, encoding: 'utf8' },
|
||||
)
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('does not take an argument')
|
||||
})
|
||||
})
|
||||
+20
-8
@@ -1,4 +1,4 @@
|
||||
/** Run the complete repository build and bind its client artifacts to their public environment. */
|
||||
/** Build repository artifacts and bind client outputs to their public environment. */
|
||||
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { rmSync } from 'node:fs'
|
||||
@@ -14,9 +14,9 @@ import {
|
||||
} from './client-build-environment.ts'
|
||||
import { pnpmInvocation } from './pnpm-invocation.ts'
|
||||
|
||||
/** Run one package script through the package manager that invoked this build. */
|
||||
function runScript(script: string, environment: NodeJS.ProcessEnv): void {
|
||||
const invocation = pnpmInvocation(['run', script], environment)
|
||||
/** Run one package-manager command with the selected public build environment. */
|
||||
function runCommand(args: readonly string[], environment: NodeJS.ProcessEnv, label: string): void {
|
||||
const invocation = pnpmInvocation(args, environment)
|
||||
const result = spawnSync(invocation.command, invocation.args, {
|
||||
cwd: resolve(import.meta.dirname, '..'),
|
||||
env: environment,
|
||||
@@ -24,18 +24,26 @@ function runScript(script: string, environment: NodeJS.ProcessEnv): void {
|
||||
})
|
||||
if (result.error !== undefined) throw result.error
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`build: ${script} exited with ${String(result.status ?? result.signal)}`)
|
||||
throw new Error(`build: ${label} exited with ${String(result.status ?? result.signal)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Run the full build selected by `--profile` or `DSH_BUILD_CLIENT_PROFILE`. */
|
||||
/** Run one package script through the package manager that invoked this build. */
|
||||
function runScript(script: string, environment: NodeJS.ProcessEnv): void {
|
||||
runCommand(['run', script], environment, script)
|
||||
}
|
||||
|
||||
/** Build client artifacts and optionally omit the repository test and script typechecks. */
|
||||
function main(): void {
|
||||
// tsdown.config.ts loads only through Node type stripping (`--config-loader native`); this names the cause before tsdown fails.
|
||||
if (!process.features.typescript) {
|
||||
throw new Error('build: Node.js TypeScript type stripping is unavailable in this Node.js process; remove --no-experimental-strip-types from NODE_OPTIONS or use a Node.js build with TypeScript support')
|
||||
}
|
||||
const { values } = parseArgs({
|
||||
options: { profile: { type: 'string' } },
|
||||
options: {
|
||||
profile: { type: 'string' },
|
||||
'artifacts-only': { type: 'boolean' },
|
||||
},
|
||||
allowPositionals: false,
|
||||
})
|
||||
const root = resolve(import.meta.dirname, '..')
|
||||
@@ -46,7 +54,11 @@ function main(): void {
|
||||
|
||||
rmSync(resolve(root, CLIENT_BUILD_RECORD_PATH), { force: true })
|
||||
runScript('build:native-system', buildEnvironment)
|
||||
runScript('build:lib', buildEnvironment)
|
||||
if (values['artifacts-only']) {
|
||||
runCommand(['exec', 'tsx', 'scripts/compile-referenced-projects.ts', 'libraries'], buildEnvironment, 'artifact libraries')
|
||||
} else {
|
||||
runScript('build:lib', buildEnvironment)
|
||||
}
|
||||
runScript('build:web', buildEnvironment)
|
||||
const record = writeClientBuildRecord(root, clientEnvironment)
|
||||
console.log(
|
||||
|
||||
@@ -14,6 +14,131 @@ const runnerPrivatePnpmDestination = /^\$\{\{ runner\.temp \}\}\/setup-pnpm-\$\{
|
||||
const nativeWindowsPnpmDestination = '${{ runner.temp }}/setup-pnpm-js-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}'
|
||||
|
||||
describe('CI workflow', () => {
|
||||
it('keeps at most ten worker jobs active while runtime targets follow the SDK suite', () => {
|
||||
const ci = loadWorkflow('.github/workflows/ci.yml')
|
||||
const compatibility = workflowJob(ci, 'node-compat')
|
||||
const runtime = workflowJob(ci, 'python-runtime')
|
||||
if (!isRecord(ci.jobs) || !isRecord(compatibility.strategy) || !isRecord(runtime.with)) {
|
||||
throw new TypeError('CI must define jobs, compatibility scheduling, and runtime inputs')
|
||||
}
|
||||
const compatibilityLimit = compatibility.strategy['max-parallel']
|
||||
const targets = runtime.with.targets
|
||||
if (typeof compatibilityLimit !== 'number' || typeof targets !== 'string') {
|
||||
throw new TypeError('CI requires a numeric compatibility limit and explicit runtime targets')
|
||||
}
|
||||
expect(Number.isInteger(compatibilityLimit)).toBe(true)
|
||||
expect(compatibilityLimit).toBeGreaterThan(0)
|
||||
expect(compatibility.strategy).toMatchObject({
|
||||
'fail-fast': false,
|
||||
matrix: { include: [{ node: '22.19' }, { node: '24.9' }, { node: 26 }] },
|
||||
})
|
||||
expect(workflowJob(ci, 'all-checks-passed').needs).toContain('node-compat')
|
||||
const ordinaryJobs = Object.entries(ci.jobs)
|
||||
.filter(([name]) => !['node-compat', 'python-runtime', 'all-checks-passed'].includes(name))
|
||||
for (const [name, job] of ordinaryJobs) {
|
||||
if (!isRecord(job)) throw new TypeError(`${name} must define a job`)
|
||||
expect(job).not.toHaveProperty('strategy')
|
||||
expect(job).not.toHaveProperty('uses')
|
||||
}
|
||||
expect(ordinaryJobs.map(([name]) => name)).toContain('python-sdk')
|
||||
expect(runtime.needs).toBe('python-sdk')
|
||||
expect(runtime.strategy).toBeUndefined()
|
||||
expect(runtime.with).toMatchObject({ targets: 'node24-linux-x64,node24-win-x64' })
|
||||
expect(runtime.uses).toBe('./.github/workflows/build-exe-for-python-sdk.yml')
|
||||
|
||||
const builder = loadWorkflow('.github/workflows/build-exe-for-python-sdk.yml')
|
||||
if (!isRecord(builder.jobs)) throw new TypeError('Python runtime builder must define jobs')
|
||||
expect(Object.keys(builder.jobs).sort()).toEqual(['build', 'plan', 'sdk-wheel'])
|
||||
const plan = workflowJob(builder, 'plan')
|
||||
const wheel = workflowJob(builder, 'sdk-wheel')
|
||||
const build = workflowJob(builder, 'build')
|
||||
expect(plan.needs).toBeUndefined()
|
||||
expect(plan.strategy).toBeUndefined()
|
||||
expect(plan.uses).toBeUndefined()
|
||||
expect(wheel.needs).toBe('plan')
|
||||
expect(wheel.strategy).toBeUndefined()
|
||||
expect(wheel.uses).toBeUndefined()
|
||||
expect(build.needs).toEqual(['plan', 'sdk-wheel'])
|
||||
expect(build.uses).toBeUndefined()
|
||||
if (!isRecord(build.strategy)) throw new TypeError('Python runtime build must define its target matrix')
|
||||
expect(build.strategy.matrix).toEqual({ include: '${{ fromJSON(needs.plan.outputs.matrix) }}' })
|
||||
|
||||
const beforeSdk = ordinaryJobs.length + compatibilityLimit
|
||||
const afterSdk = ordinaryJobs.length - 1 + compatibilityLimit + Math.max(1, targets.split(',').length)
|
||||
expect(beforeSdk).toBeLessThanOrEqual(10)
|
||||
expect(afterSdk).toBeLessThanOrEqual(10)
|
||||
})
|
||||
|
||||
it('retains aggregate diagnostics in required builds and protected publication', () => {
|
||||
const ci = loadWorkflow('.github/workflows/ci.yml')
|
||||
for (const [name, command] of [
|
||||
['node-24-consumers', 'pnpm run check:ci:consumers'],
|
||||
['windows-build', 'pnpm run check:ci:windows-blocking'],
|
||||
] as const) {
|
||||
const job = workflowJob(ci, name)
|
||||
expect(job.steps).toContainEqual(expect.objectContaining({ run: command }))
|
||||
}
|
||||
for (const [file, command] of [
|
||||
['release-publish.yml', 'pnpm run build:official'],
|
||||
['release-vendor-publish.yml', 'pnpm run build:lib:host'],
|
||||
] as const) {
|
||||
expect(workflowJob(loadWorkflow('.github/workflows/' + file), 'pack').steps)
|
||||
.toContainEqual(expect.objectContaining({ name: 'Build', run: command }))
|
||||
}
|
||||
const scripts = JSON.parse(readFileSync(resolve(root, 'package.json'), 'utf8')) as { scripts: Record<string, string> }
|
||||
expect(scripts.scripts['build:lib:host']).toContain('tsc -b tsconfig.host.json')
|
||||
expect(scripts.scripts['build:lib:client']).toContain('tsc -b tsconfig.client.json')
|
||||
expect(scripts.scripts.typecheck).toBe('npm run build:lib:host && npm run typecheck:contracts-ready')
|
||||
})
|
||||
|
||||
it('selects artifact builds only for PR release and API rehearsals', () => {
|
||||
const official = "${{ github.event_name == 'pull_request' && 'pnpm run build --artifacts-only --profile official' || 'pnpm run build:official' }}"
|
||||
const vendor = "${{ github.event_name == 'pull_request' && 'pnpm exec tsx scripts/compile-referenced-projects.ts host-libraries' || 'pnpm run build:lib:host' }}"
|
||||
for (const [file, jobName, expected] of [
|
||||
['release.yml', 'pack', official],
|
||||
['release-vendor.yml', 'pack', vendor],
|
||||
['e2e.yml', 'e2e', official],
|
||||
] as const) {
|
||||
const job = workflowJob(loadWorkflow('.github/workflows/' + file), jobName)
|
||||
if (!Array.isArray(job.steps)) throw new TypeError(`${jobName} must define steps`)
|
||||
const step = job.steps.filter(isRecord).find(candidate => typeof candidate.run === 'string'
|
||||
&& (candidate.run.includes('--artifacts-only') || candidate.run.includes('host-libraries')))
|
||||
expect(step?.run).toBe(expected)
|
||||
for (const event of ['pull_request', 'push', 'workflow_dispatch']) {
|
||||
const expression = expected.slice('${{ '.length, -' }}'.length)
|
||||
const selected: unknown = runInNewContext(expression, { github: { event_name: event } }, { timeout: 1000 })
|
||||
expect(selected).toBe(event === 'pull_request'
|
||||
? file === 'release-vendor.yml'
|
||||
? 'pnpm exec tsx scripts/compile-referenced-projects.ts host-libraries'
|
||||
: 'pnpm run build --artifacts-only --profile official'
|
||||
: file === 'release-vendor.yml' ? 'pnpm run build:lib:host' : 'pnpm run build:official')
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('selects PR artifact compilation inside the Python executable builder', () => {
|
||||
const job = workflowJob(loadWorkflow('.github/workflows/build-exe-for-python-sdk.yml'), 'build')
|
||||
if (!Array.isArray(job.steps)) throw new TypeError('Python runtime builder must define steps')
|
||||
const steps = job.steps.filter(isRecord)
|
||||
const execute = steps.find(step => step.name === 'Build single-exe')
|
||||
expect(steps.some(step => step.run === 'pnpm run build --artifacts-only')).toBe(false)
|
||||
expect(execute).toMatchObject({
|
||||
env: { DSH_BUILD_CLIENT_PROFILE: 'official' },
|
||||
})
|
||||
if (typeof execute?.run !== 'string') throw new TypeError('Python executable builder must define a command')
|
||||
const expression = [...execute.run.matchAll(/\$\{\{ ([^{}]+) \}\}/g)]
|
||||
.map(match => match[1]).find(value => value?.startsWith('inputs.ci'))
|
||||
expect(expression).toBe("inputs.ci && github.event_name == 'pull_request' && '--artifacts-only' || ''")
|
||||
if (expression === undefined) throw new TypeError('Python executable builder must select its build mode')
|
||||
for (const ci of [false, true]) {
|
||||
for (const event of ['pull_request', 'push', 'workflow_dispatch']) {
|
||||
const context = { inputs: { ci }, github: { event_name: event } }
|
||||
const selected: unknown = runInNewContext(expression, context, { timeout: 1000 })
|
||||
expect(selected).toBe(ci && event === 'pull_request' ? '--artifacts-only' : '')
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('prepares confinement before Node compatibility smokes', () => {
|
||||
const job = workflowJob(loadWorkflow('.github/workflows/ci.yml'), 'node-compat')
|
||||
if (!Array.isArray(job.steps)) throw new TypeError('Node compatibility job must define steps')
|
||||
@@ -62,6 +187,34 @@ describe('CI workflow', () => {
|
||||
expect(wine.steps).toContainEqual(expect.objectContaining({ name: 'Shut down wineserver', if: 'always()' }))
|
||||
})
|
||||
|
||||
it.each([
|
||||
['node-24-coverage', 'Linux', 'LINUX', 'pnpm run check:ci:coverage'],
|
||||
['windows-coverage', 'Windows', 'WINDOWS', 'pnpm run check:ci:coverage'],
|
||||
])('reuses %s coverage timings only within its platform and runner pool', (jobName, platform, variable, command) => {
|
||||
const job = workflowJob(loadWorkflow('.github/workflows/ci.yml'), jobName)
|
||||
if (!Array.isArray(job.steps)) throw new TypeError(`${jobName} must define steps`)
|
||||
const steps = job.steps.filter(isRecord)
|
||||
const checkout = steps.findIndex(step => step.uses === 'actions/checkout@v6')
|
||||
const restore = steps.findIndex(step => step.name === 'Restore coverage duration history')
|
||||
const coverage = steps.findIndex(step => step.run === command)
|
||||
const save = steps.findIndex(step => step.name === 'Save coverage duration history')
|
||||
const prefix = `coverage-times-${platform}-\${{ runner.environment }}-\${{ vars.DSH_CI_FAILOVER_${variable} || 'enterprise' }}-`
|
||||
const key = prefix + '${{ github.run_id }}-${{ github.run_attempt }}'
|
||||
expect(checkout).toBeGreaterThanOrEqual(0)
|
||||
expect(restore).toBeGreaterThan(checkout)
|
||||
expect(coverage).toBeGreaterThan(restore)
|
||||
expect(save).toBeGreaterThan(coverage)
|
||||
expect(steps[restore]).toMatchObject({
|
||||
uses: 'actions/cache/restore@v4',
|
||||
with: { path: '.coverage-times.json', key, 'restore-keys': `${prefix}\n` },
|
||||
})
|
||||
expect(steps[save]).toMatchObject({
|
||||
uses: 'actions/cache/save@v4',
|
||||
if: '${{ !cancelled() }}',
|
||||
with: { path: '.coverage-times.json', key },
|
||||
})
|
||||
})
|
||||
|
||||
it('isolates every pnpm action setup destination per runner', () => {
|
||||
const files = ['.github/workflows/ci.yml', '.github/workflows/ci-master.yml']
|
||||
const setups: Array<{ jobName: string; step: unknown }> = []
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { globSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url'
|
||||
import { describe, expect, it, onTestFinished, vi } from 'vitest'
|
||||
import { buildLibraryArtifacts, compileReferencedProjects } from './compile-referenced-projects.ts'
|
||||
import { runCiBench } from './run-ci-bench.ts'
|
||||
import { removeFixtureSafely } from './test-fixture-cleanup.ts'
|
||||
|
||||
const compiler = fileURLToPath(import.meta.resolve('typescript/bin/tsc'))
|
||||
|
||||
function fixture(): { root: string; leaf: string } {
|
||||
const root = mkdtempSync(join(tmpdir(), 'dsh-referenced-projects-'))
|
||||
onTestFinished(() => { removeFixtureSafely(root) })
|
||||
const leaf = join(root, 'leaf package')
|
||||
mkdirSync(join(leaf, 'src'), { recursive: true })
|
||||
writeFileSync(join(root, 'package.json'), JSON.stringify({ type: 'module' }))
|
||||
writeFileSync(join(leaf, 'src/index.ts'), 'export const value: number = 42\n')
|
||||
const compilerOptions = {
|
||||
target: 'ES2024',
|
||||
module: 'NodeNext',
|
||||
moduleResolution: 'NodeNext',
|
||||
lib: ['ES2024'],
|
||||
types: [],
|
||||
composite: true,
|
||||
strict: true,
|
||||
}
|
||||
writeFileSync(join(leaf, 'tsconfig.json'), JSON.stringify({
|
||||
compilerOptions: {
|
||||
...compilerOptions,
|
||||
rootDir: 'src',
|
||||
outDir: 'lib/types',
|
||||
declaration: true,
|
||||
declarationMap: true,
|
||||
sourceMap: true,
|
||||
},
|
||||
include: ['src'],
|
||||
}))
|
||||
writeFileSync(join(root, 'aggregate.ts'), 'import { value } from "./leaf package/src/index.js"\nexport const aggregate: number = value\n')
|
||||
for (const face of ['host', 'client']) {
|
||||
writeFileSync(join(root, `tsconfig.${face}.json`), JSON.stringify({
|
||||
compilerOptions: { ...compilerOptions, noEmit: true },
|
||||
include: ['aggregate.ts'],
|
||||
references: [{ path: './leaf package' }],
|
||||
}))
|
||||
}
|
||||
return { root, leaf }
|
||||
}
|
||||
|
||||
function artifacts(leaf: string): Record<string, string> {
|
||||
const output = join(leaf, 'lib')
|
||||
return Object.fromEntries(globSync('**/*', { cwd: output })
|
||||
.filter(path => statSync(join(output, path)).isFile())
|
||||
.sort()
|
||||
.map(path => [path.replaceAll('\\', '/'), readFileSync(join(output, path)).toString('base64')]))
|
||||
}
|
||||
|
||||
function recordedCommands(root: string, failHostBundle = false): () => { args: string[]; title: string }[] {
|
||||
const log = join(root, 'commands.jsonl')
|
||||
const entrypoint = join(root, 'pnpm', 'bin', 'pnpm.cjs')
|
||||
mkdirSync(join(root, 'pnpm', 'bin'), { recursive: true })
|
||||
writeFileSync(entrypoint, [
|
||||
'const { appendFileSync } = require("node:fs")',
|
||||
'const args = process.argv.slice(2)',
|
||||
`appendFileSync(${JSON.stringify(log)}, JSON.stringify({ args, title: process.env.DSH_CLIENT_TITLE }) + "\\n")`,
|
||||
...failHostBundle ? ['if (args.includes("host")) process.exit(7)'] : [],
|
||||
].join('\n') + '\n')
|
||||
vi.stubEnv('npm_execpath', entrypoint)
|
||||
vi.stubEnv('DSH_CLIENT_TITLE', 'Inherited Build Title')
|
||||
onTestFinished(() => { vi.unstubAllEnvs() })
|
||||
return () => readFileSync(log, 'utf8').trim().split('\n')
|
||||
.map(line => JSON.parse(line) as { args: string[]; title: string })
|
||||
}
|
||||
|
||||
describe('internal artifact build order', () => {
|
||||
it('resolves pnpm for a direct CLI without npm_execpath and preserves its environment', () => {
|
||||
const { root } = fixture()
|
||||
const marker = join(root, 'direct-cli-title.txt')
|
||||
const environment: NodeJS.ProcessEnv = { ...process.env, DSH_CLIENT_TITLE: 'Direct CLI Title' }
|
||||
delete environment.npm_execpath
|
||||
const helper = pathToFileURL(resolve(import.meta.dirname, 'compile-referenced-projects.ts')).href
|
||||
const code = `import { runPnpmCommand } from ${JSON.stringify(helper)}; runPnpmCommand(${JSON.stringify(root)}, ${JSON.stringify([
|
||||
'exec', process.execPath, '-e',
|
||||
`require('node:fs').writeFileSync(${JSON.stringify(marker)}, process.env.DSH_CLIENT_TITLE)`,
|
||||
])})`
|
||||
const result = spawnSync(process.execPath, [
|
||||
'--import', import.meta.resolve('tsx/esm'),
|
||||
'--input-type=module', '-e', code,
|
||||
], { cwd: root, env: environment, encoding: 'utf8' })
|
||||
|
||||
expect(result.status, result.stdout + result.stderr).toBe(0)
|
||||
expect(readFileSync(marker, 'utf8')).toBe('Direct CLI Title')
|
||||
})
|
||||
|
||||
it.each([true, false])('preserves bundler order and the public environment with hostOnly=%s', (hostOnly) => {
|
||||
const { root } = fixture()
|
||||
const commands = recordedCommands(root)
|
||||
|
||||
buildLibraryArtifacts(root, hostOnly)
|
||||
|
||||
const expected = [
|
||||
['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'host'],
|
||||
['--filter', '@deepseek-ai/dsh-desktop', 'run', 'bundle'],
|
||||
...hostOnly ? [] : [['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'client']],
|
||||
]
|
||||
expect(commands()).toEqual(expected.map(args => ({ args, title: 'Inherited Build Title' })))
|
||||
})
|
||||
|
||||
it('stops after a failed Host bundler before Desktop and Client builds', () => {
|
||||
const { root } = fixture()
|
||||
const commands = recordedCommands(root, true)
|
||||
|
||||
expect(() => { buildLibraryArtifacts(root, false) }).toThrow('exited with 7')
|
||||
expect(commands()).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('defers benchmark diagnostics while normal artifact builds reject the same source', () => {
|
||||
const { root, leaf } = fixture()
|
||||
const commands = recordedCommands(root)
|
||||
writeFileSync(join(leaf, 'src/index.ts'), 'export const value: number = "invalid"\n')
|
||||
|
||||
runCiBench(root)
|
||||
|
||||
expect(commands()).toEqual([
|
||||
['run', 'build:native-system'],
|
||||
['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'host'],
|
||||
['--filter', '@deepseek-ai/dsh-desktop', 'run', 'bundle'],
|
||||
['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'client'],
|
||||
['exec', 'tsdown', '--config-loader', 'native', '--config', 'benchmarks/tsdown.config.ts'],
|
||||
['run', 'build:web'],
|
||||
['run', 'test:bench:built'],
|
||||
].map(args => ({ args, title: 'Inherited Build Title' })))
|
||||
|
||||
expect(() => { buildLibraryArtifacts(root, false) }).toThrow('host compiler exited with')
|
||||
expect(commands()).toHaveLength(7)
|
||||
})
|
||||
})
|
||||
|
||||
describe('package project compilation', () => {
|
||||
it('records Windows-style glob keys without changing artifact bytes', () => {
|
||||
const { leaf } = fixture()
|
||||
const output = join(leaf, 'lib')
|
||||
mkdirSync(join(output, 'types'), { recursive: true })
|
||||
const bytes = Buffer.from([0, 255, 13, 10])
|
||||
writeFileSync(join(output, 'types\\index.js'), bytes)
|
||||
|
||||
expect(artifacts(leaf)).toEqual({ 'types/index.js': bytes.toString('base64') })
|
||||
})
|
||||
|
||||
it.each(['host', 'client'] as const)('preserves every %s package artifact byte', (face) => {
|
||||
const { root, leaf } = fixture()
|
||||
const complete = spawnSync(process.execPath, [compiler, '-b', `tsconfig.${face}.json`], {
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
})
|
||||
expect(complete.status, complete.stdout + complete.stderr).toBe(0)
|
||||
const expected = artifacts(leaf)
|
||||
expect(Object.keys(expected)).toEqual(expect.arrayContaining([
|
||||
'types/index.js', 'types/index.js.map', 'types/index.d.ts', 'types/index.d.ts.map',
|
||||
]))
|
||||
rmSync(join(leaf, 'lib'), { recursive: true })
|
||||
|
||||
compileReferencedProjects(root, face)
|
||||
|
||||
expect(artifacts(leaf)).toEqual(expected)
|
||||
})
|
||||
|
||||
it.each(['host', 'client'] as const)('preserves %s runtime output during benchmark emission', (face) => {
|
||||
const { root, leaf } = fixture()
|
||||
writeFileSync(join(leaf, 'src/support.ts'), [
|
||||
'export interface Payload { label: string }',
|
||||
'export const enum Offset { Baseline = 40 }',
|
||||
'',
|
||||
].join('\n'))
|
||||
writeFileSync(join(leaf, 'src/index.ts'), [
|
||||
'import { Offset, type Payload } from "./support.js"',
|
||||
'export const value: number = Offset.Baseline + 2',
|
||||
'export function describe(payload: Payload): string { return `${payload.label}: ${value}` }',
|
||||
'',
|
||||
].join('\n'))
|
||||
compileReferencedProjects(root, face)
|
||||
const runtimeArtifacts = () => Object.fromEntries(Object.entries(artifacts(leaf))
|
||||
.filter(([path]) => /\.js(?:\.map)?$/.test(path)))
|
||||
const expected = runtimeArtifacts()
|
||||
expect(Object.keys(expected)).toEqual([
|
||||
'types/index.js', 'types/index.js.map', 'types/support.js', 'types/support.js.map',
|
||||
])
|
||||
rmSync(join(leaf, 'lib'), { recursive: true })
|
||||
|
||||
compileReferencedProjects(root, face, 'benchmark-emit')
|
||||
|
||||
expect(runtimeArtifacts()).toEqual(expected)
|
||||
const entrypoint = pathToFileURL(join(leaf, 'lib/types/index.js')).href
|
||||
const result = spawnSync(process.execPath, [
|
||||
'--input-type=module', '-e',
|
||||
`import { describe } from ${JSON.stringify(entrypoint)}; console.log(describe({ label: 'answer' }))`,
|
||||
], { cwd: root, encoding: 'utf8' })
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.signal).toBeNull()
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
expect(result.stdout.trim()).toBe('answer: 42')
|
||||
})
|
||||
|
||||
it.each(['host', 'client'] as const)('rechecks %s semantic errors after benchmark emission without forcing a rebuild', (face) => {
|
||||
const { root, leaf } = fixture()
|
||||
writeFileSync(join(leaf, 'src/index.ts'), 'export const value: number = "invalid"\n')
|
||||
|
||||
compileReferencedProjects(root, face, 'benchmark-emit')
|
||||
|
||||
expect(Object.keys(artifacts(leaf)).some(path => path.endsWith('.tsbuildinfo'))).toBe(true)
|
||||
const complete = spawnSync(process.execPath, [compiler, '-b', `tsconfig.${face}.json`], {
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
})
|
||||
expect(complete.error).toBeUndefined()
|
||||
expect(complete.signal).toBeNull()
|
||||
expect(complete.status).not.toBe(0)
|
||||
expect(complete.stdout).toContain('src/index.ts(1,14): error TS2322')
|
||||
})
|
||||
|
||||
it('rejects package syntax errors during benchmark emission', () => {
|
||||
const { root, leaf } = fixture()
|
||||
writeFileSync(join(leaf, 'src/index.ts'), 'export const value = ;\n')
|
||||
|
||||
expect(() => { compileReferencedProjects(root, 'host', 'benchmark-emit') })
|
||||
.toThrow('host compiler exited with')
|
||||
})
|
||||
|
||||
it('keeps aggregate-only errors in the complete compiler check', () => {
|
||||
const { root } = fixture()
|
||||
writeFileSync(join(root, 'aggregate.ts'), 'export const aggregate: number = "invalid"\n')
|
||||
|
||||
expect(() => { compileReferencedProjects(root, 'host') }).not.toThrow()
|
||||
const complete = spawnSync(process.execPath, [compiler, '-b', 'tsconfig.host.json'], {
|
||||
cwd: root,
|
||||
encoding: 'utf8',
|
||||
})
|
||||
expect(complete.status).not.toBe(0)
|
||||
expect(complete.stdout).toContain('aggregate.ts(1,14): error TS2322')
|
||||
})
|
||||
|
||||
it('rejects package source errors', () => {
|
||||
const { root, leaf } = fixture()
|
||||
writeFileSync(join(leaf, 'src/index.ts'), 'export const value: number = "invalid"\n')
|
||||
|
||||
expect(() => { compileReferencedProjects(root, 'host') }).toThrow('host compiler exited with')
|
||||
})
|
||||
|
||||
it.each([undefined, []])('rejects an aggregate with missing or empty references: %j', (references) => {
|
||||
const { root } = fixture()
|
||||
writeFileSync(join(root, 'tsconfig.host.json'), JSON.stringify({ files: [], references }))
|
||||
|
||||
expect(() => { compileReferencedProjects(root, 'host') }).toThrow('requires non-empty project references')
|
||||
})
|
||||
|
||||
it('rejects a missing referenced project', () => {
|
||||
const { root } = fixture()
|
||||
writeFileSync(join(root, 'tsconfig.host.json'), JSON.stringify({
|
||||
compilerOptions: { noEmit: true },
|
||||
files: [],
|
||||
references: [{ path: './absent' }],
|
||||
}))
|
||||
|
||||
expect(() => { compileReferencedProjects(root, 'host') }).toThrow('host compiler exited with')
|
||||
})
|
||||
|
||||
it.each([undefined, false])('rejects an aggregate capable of emitting files: noEmit=%s', (noEmit) => {
|
||||
const { root } = fixture()
|
||||
writeFileSync(join(root, 'tsconfig.host.json'), JSON.stringify({
|
||||
compilerOptions: { noEmit },
|
||||
include: ['aggregate.ts'],
|
||||
references: [{ path: './leaf package' }],
|
||||
}))
|
||||
|
||||
expect(() => { compileReferencedProjects(root, 'host') }).toThrow(
|
||||
`${join(root, 'tsconfig.host.json')} requires noEmit: true`,
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects a missing aggregate config', () => {
|
||||
const { root } = fixture()
|
||||
rmSync(join(root, 'tsconfig.host.json'))
|
||||
|
||||
expect(() => { compileReferencedProjects(root, 'host') }).toThrow('Cannot read file')
|
||||
})
|
||||
|
||||
it('rejects invalid compiler options', () => {
|
||||
const { root } = fixture()
|
||||
writeFileSync(join(root, 'tsconfig.host.json'), JSON.stringify({
|
||||
compilerOptions: { target: 'invalid' },
|
||||
files: [],
|
||||
references: [{ path: './leaf package' }],
|
||||
}))
|
||||
|
||||
expect(() => { compileReferencedProjects(root, 'host') }).toThrow("Argument for '--target' option must be")
|
||||
})
|
||||
|
||||
it('rejects an unsupported build mode before building', () => {
|
||||
const repository = resolve(import.meta.dirname, '..')
|
||||
const result = spawnSync(process.execPath, [
|
||||
'--import', 'tsx/esm',
|
||||
resolve(repository, 'scripts/compile-referenced-projects.ts'),
|
||||
'unsupported',
|
||||
], { cwd: repository, encoding: 'utf8' })
|
||||
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('expected exactly one mode: host, client, host-libraries, or libraries')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,111 @@
|
||||
/** Compile and bundle package projects without repeating aggregate test and script typechecks. */
|
||||
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { relative, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { parseArgs } from 'node:util'
|
||||
import ts from 'typescript'
|
||||
import { pnpmCommand } from './release/process.ts'
|
||||
|
||||
type CompilationMode = 'checked' | 'benchmark-emit'
|
||||
|
||||
/**
|
||||
* Run one package-manager command serially with the inherited build environment.
|
||||
* @param root - Repository root supplying the command's working directory.
|
||||
* @param args - Package-manager arguments, without shell quoting.
|
||||
* @throws If the command cannot start or exits unsuccessfully.
|
||||
*/
|
||||
export function runPnpmCommand(root: string, args: readonly string[]): void {
|
||||
const [command, ...prefix] = pnpmCommand()
|
||||
const result = spawnSync(command, [...prefix, ...args], { cwd: root, stdio: 'inherit' })
|
||||
if (result.error !== undefined) throw result.error
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`compile-referenced-projects: pnpm ${args.join(' ')} exited with ${String(result.status ?? result.signal)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile one aggregate's project references with the serial TypeScript builder.
|
||||
* @param root - Repository root containing the aggregate tsconfig.
|
||||
* @param face - Independent compiler face whose package projects are emitted.
|
||||
* @param mode - Check package types by default; benchmark emission defers diagnostics to required full builds.
|
||||
* @throws If the aggregate emits files, its configuration is invalid, or package compilation fails.
|
||||
*/
|
||||
export function compileReferencedProjects(
|
||||
root: string,
|
||||
face: 'host' | 'client',
|
||||
mode: CompilationMode = 'checked',
|
||||
): void {
|
||||
const configPath = resolve(root, `tsconfig.${face}.json`)
|
||||
const config = ts.getParsedCommandLineOfConfigFile(configPath, {}, {
|
||||
...ts.sys,
|
||||
onUnRecoverableConfigFileDiagnostic(diagnostic) {
|
||||
throw new Error(ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n'))
|
||||
},
|
||||
})
|
||||
if (config === undefined) throw new Error(`compile-referenced-projects: cannot read ${configPath}`)
|
||||
const references = config.projectReferences
|
||||
if (references === undefined || references.length === 0) {
|
||||
throw new Error(`compile-referenced-projects: ${configPath} requires non-empty project references`)
|
||||
}
|
||||
if (config.errors.length > 0) {
|
||||
throw new Error(config.errors
|
||||
.map(diagnostic => ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n'))
|
||||
.join('\n'))
|
||||
}
|
||||
if (config.options.noEmit !== true) {
|
||||
throw new Error(`compile-referenced-projects: ${configPath} requires noEmit: true to exclude its aggregate program`)
|
||||
}
|
||||
const compiler = fileURLToPath(import.meta.resolve('typescript/bin/tsc'))
|
||||
const result = spawnSync(process.execPath, [
|
||||
...face === 'host' ? ['--max-old-space-size=4096'] : [],
|
||||
compiler,
|
||||
'-b',
|
||||
...mode === 'benchmark-emit' ? ['--noCheck'] : [],
|
||||
// Relative arguments keep the complete workspace graph below Windows' command-line limit.
|
||||
...references.map(reference => relative(root, reference.path)),
|
||||
], { cwd: root, stdio: 'inherit' })
|
||||
if (result.error !== undefined) throw result.error
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`compile-referenced-projects: ${face} compiler exited with ${String(result.status ?? result.signal)}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit package libraries in Host, Desktop, then Client dependency order.
|
||||
* @param root - Repository root containing the compiler and bundle configurations.
|
||||
* @param hostOnly - Omit the Client compiler and bundler after the Host artifacts are ready.
|
||||
* @param mode - Check package types by default; benchmark emission defers diagnostics to required full builds.
|
||||
* @throws If any compiler or bundler fails.
|
||||
*/
|
||||
export function buildLibraryArtifacts(root: string, hostOnly: boolean, mode: CompilationMode = 'checked'): void {
|
||||
compileReferencedProjects(root, 'host', mode)
|
||||
runPnpmCommand(root, ['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'host'])
|
||||
runPnpmCommand(root, ['--filter', '@deepseek-ai/dsh-desktop', 'run', 'bundle'])
|
||||
if (hostOnly) return
|
||||
compileReferencedProjects(root, 'client', mode)
|
||||
runPnpmCommand(root, ['exec', 'tsdown', '--config-loader', 'native', '--env.DSH_BUILD_FACE', 'client'])
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const { positionals } = parseArgs({ allowPositionals: true })
|
||||
const [mode] = positionals
|
||||
if (positionals.length !== 1) {
|
||||
throw new Error('compile-referenced-projects: expected exactly one mode: host, client, host-libraries, or libraries')
|
||||
}
|
||||
const root = resolve(import.meta.dirname, '..')
|
||||
switch (mode) {
|
||||
case 'host':
|
||||
case 'client':
|
||||
compileReferencedProjects(root, mode)
|
||||
break
|
||||
case 'host-libraries':
|
||||
buildLibraryArtifacts(root, true)
|
||||
break
|
||||
case 'libraries':
|
||||
buildLibraryArtifacts(root, false)
|
||||
break
|
||||
default:
|
||||
throw new Error('compile-referenced-projects: expected exactly one mode: host, client, host-libraries, or libraries')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import type { ModuleDiagnostic, ResolvedConfig } from 'vitest/node'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import CoverageFileTimesReporter from './coverage-file-times.ts'
|
||||
|
||||
const roots: string[] = []
|
||||
afterEach(async () => {
|
||||
for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function temporaryRoot(): Promise<string> {
|
||||
const root = await mkdtemp(join(tmpdir(), 'dsh-coverage-file-times-'))
|
||||
roots.push(root)
|
||||
return root
|
||||
}
|
||||
|
||||
describe('coverage file timing report', () => {
|
||||
it('includes import and hook costs once without serializing coverage or test objects', async () => {
|
||||
const root = await temporaryRoot()
|
||||
const reporter = new CoverageFileTimesReporter()
|
||||
reporter.onInit({ config: { root, outputFile: { fileTimes: 'times.json' } } })
|
||||
const diagnostic: ModuleDiagnostic = {
|
||||
environmentSetupDuration: 3,
|
||||
prepareDuration: 5,
|
||||
setupDuration: 7,
|
||||
collectDuration: 100,
|
||||
duration: 200,
|
||||
heap: undefined,
|
||||
// Per-dependency import times are already included in collectDuration.
|
||||
importDurations: { dependency: { selfTime: 100, totalTime: 100 } },
|
||||
}
|
||||
const module = {
|
||||
moduleId: join(root, 'packages', 'a', 'tests', 'x.spec.ts'),
|
||||
diagnostic: () => diagnostic,
|
||||
// Vitest modules retain tests and coverage through their project. The
|
||||
// timing report must read only their paths and diagnostics.
|
||||
toJSON: () => { throw new Error('test module was serialized') },
|
||||
}
|
||||
await reporter.onTestRunEnd([module])
|
||||
|
||||
const output = await readFile(join(root, 'times.json'), 'utf8')
|
||||
expect(JSON.parse(output)).toEqual({ 'packages/a/tests/x.spec.ts': 315 })
|
||||
expect(output.endsWith('\n')).toBe(true)
|
||||
expect(output).not.toContain('coverageMap')
|
||||
expect(output).not.toContain('assertionResults')
|
||||
})
|
||||
|
||||
it('writes an empty report when no modules finished', async () => {
|
||||
const root = await temporaryRoot()
|
||||
const reporter = new CoverageFileTimesReporter()
|
||||
reporter.onInit({ config: { root, outputFile: { fileTimes: 'times.json' } } })
|
||||
await reporter.onTestRunEnd([])
|
||||
expect(await readFile(join(root, 'times.json'), 'utf8')).toBe('{}\n')
|
||||
})
|
||||
|
||||
const missingOutputs: ResolvedConfig['outputFile'][] = ['report.json', {}, { fileTimes: '' }]
|
||||
it.each(missingOutputs)(
|
||||
'rejects missing timing output %j',
|
||||
(outputFile) => {
|
||||
const reporter = new CoverageFileTimesReporter()
|
||||
expect(() => { reporter.onInit({ config: { root: '/repo', outputFile } }) })
|
||||
.toThrow('outputFile.fileTimes is required')
|
||||
},
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,43 @@
|
||||
/** Record coverage partition file costs without retaining test results or coverage maps. */
|
||||
import { writeFile } from 'node:fs/promises'
|
||||
import { relative, resolve, sep } from 'node:path'
|
||||
import type { Reporter, ResolvedConfig, TestModule } from 'vitest/node'
|
||||
|
||||
/** Vitest reporter that writes repository-relative file durations to `outputFile.fileTimes`. */
|
||||
export default class CoverageFileTimesReporter implements Reporter {
|
||||
private root = ''
|
||||
private outputFile = ''
|
||||
|
||||
/**
|
||||
* Resolve the timing output owned by the coverage coordinator.
|
||||
* @param vitest - repository root and configured reporter output files.
|
||||
* @throws when the file-times output is absent.
|
||||
*/
|
||||
public onInit(vitest: { config: Pick<ResolvedConfig, 'root' | 'outputFile'> }): void {
|
||||
const output = vitest.config.outputFile
|
||||
if (typeof output !== 'object' || typeof output.fileTimes !== 'string' || output.fileTimes === '') {
|
||||
throw new Error('coverage-file-times: outputFile.fileTimes is required.')
|
||||
}
|
||||
this.root = vitest.config.root
|
||||
this.outputFile = resolve(this.root, output.fileTimes)
|
||||
}
|
||||
|
||||
/**
|
||||
* Write each file's distinct environment, preparation, setup, collection, and execution costs.
|
||||
* @param testModules - finished files, including failed and skipped files.
|
||||
* @returns after the compact timing report has been written.
|
||||
*/
|
||||
public async onTestRunEnd(testModules: readonly Pick<TestModule, 'moduleId' | 'diagnostic'>[]): Promise<void> {
|
||||
const durations: Record<string, number> = {}
|
||||
for (const module of testModules) {
|
||||
const diagnostic = module.diagnostic()
|
||||
const file = relative(this.root, module.moduleId).split(sep).join('/')
|
||||
durations[file] = diagnostic.environmentSetupDuration
|
||||
+ diagnostic.prepareDuration
|
||||
+ diagnostic.setupDuration
|
||||
+ diagnostic.collectDuration
|
||||
+ diagnostic.duration
|
||||
}
|
||||
await writeFile(this.outputFile, `${JSON.stringify(durations)}\n`, 'utf8')
|
||||
}
|
||||
}
|
||||
@@ -269,17 +269,26 @@ describe('coverage file inventory', () => {
|
||||
expect(durations.get('packages/a/tests/y.spec.ts')).toBe(7)
|
||||
})
|
||||
|
||||
it('extracts per-file durations from partition json reports', async () => {
|
||||
it('extracts per-file durations from compact partition timing reports', async () => {
|
||||
const root = await temporaryRoot()
|
||||
const report = join(root, 'partition-1.report.json')
|
||||
const report = join(root, 'partition-1.times.json')
|
||||
await writeFile(report, JSON.stringify({
|
||||
testResults: [
|
||||
{ name: join(root, 'packages/a/tests/x.spec.ts'), startTime: 1000, endTime: 1500 },
|
||||
{ name: 'not-a-spec', startTime: 1, endTime: 2 },
|
||||
],
|
||||
'packages/a/tests/x.spec.ts': 500,
|
||||
'packages/a/tests/y.spec.ts': 0,
|
||||
'packages/a/tests/invalid.spec.ts': -1,
|
||||
'packages/a/tests/malformed.spec.ts': 'unknown',
|
||||
}))
|
||||
const durations = collectPartitionDurations([report], root)
|
||||
const durations = collectPartitionDurations([report])
|
||||
expect(durations.get('packages/a/tests/x.spec.ts')).toBe(500)
|
||||
expect(durations.get('packages/a/tests/y.spec.ts')).toBe(0)
|
||||
expect(durations.size).toBe(2)
|
||||
})
|
||||
|
||||
it.each(['null', '[]', '42', '"invalid"', '{'])('ignores corrupt timing metadata %s', async (contents) => {
|
||||
const root = await temporaryRoot()
|
||||
const report = join(root, 'partition.times.json')
|
||||
await writeFile(report, contents)
|
||||
expect(collectPartitionDurations([report])).toEqual(new Map())
|
||||
})
|
||||
})
|
||||
|
||||
@@ -511,9 +520,10 @@ describe('coverage partition coordinator', () => {
|
||||
'--maxWorkers=1',
|
||||
'--reporter=default',
|
||||
'--reporter=blob',
|
||||
'--reporter=json',
|
||||
'--testTimeout=30000',
|
||||
]))
|
||||
expect(command.args).not.toContain('--reporter=json')
|
||||
expect(command.args).toContain('--reporter=./scripts/coverage-file-times.ts')
|
||||
expect(command.args).not.toContain('--shard=1/3')
|
||||
expect(command.args.some(argument => argument.startsWith('--config='))).toBe(true)
|
||||
expect(command.env).toEqual({
|
||||
|
||||
@@ -27,6 +27,7 @@ export const COVERAGE_TEST_TIMEOUT_ENV = 'DSH_COVERAGE_TEST_TIMEOUT_MS'
|
||||
* working directory on every platform.
|
||||
*/
|
||||
const CANONICAL_LOCATIONS_REPORTER = './scripts/coverage-canonical-locations.ts'
|
||||
const FILE_TIMES_REPORTER = './scripts/coverage-file-times.ts'
|
||||
|
||||
/** One child command owned by the coverage coordinator. */
|
||||
export interface CoverageCommand {
|
||||
@@ -299,23 +300,23 @@ export function writeFileDurations(
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract per-file durations from Vitest JSON reporter outputs (one per
|
||||
* partition). Each `testResults` entry names an absolute spec path and carries
|
||||
* `startTime`/`endTime`; the difference is the file's recorded duration.
|
||||
* Extract per-file durations from compact timing reports, one per partition.
|
||||
* @param reportFiles - reports mapping repository-relative file paths to milliseconds.
|
||||
* @returns finite, non-negative file costs from readable reports.
|
||||
*/
|
||||
export function collectPartitionDurations(reportFiles: readonly string[], root: string): Map<string, number> {
|
||||
export function collectPartitionDurations(reportFiles: readonly string[]): Map<string, number> {
|
||||
const durations = new Map<string, number>()
|
||||
for (const file of reportFiles) {
|
||||
let report: { testResults?: Array<{ name?: unknown; startTime?: number; endTime?: number }> }
|
||||
let report: unknown
|
||||
try {
|
||||
report = JSON.parse(readFileSync(file, 'utf8')) as { testResults?: Array<{ name?: unknown; startTime?: number; endTime?: number }> }
|
||||
report = JSON.parse(readFileSync(file, 'utf8'))
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
for (const result of report.testResults ?? []) {
|
||||
if (typeof result.name !== 'string' || typeof result.startTime !== 'number' || typeof result.endTime !== 'number') continue
|
||||
const relativePath = relative(root, result.name).split(sep).join('/')
|
||||
durations.set(relativePath, Math.max(0, result.endTime - result.startTime))
|
||||
if (report === null || typeof report !== 'object' || Array.isArray(report)) continue
|
||||
for (const [path, duration] of Object.entries(report)) {
|
||||
if (typeof duration !== 'number' || !Number.isFinite(duration) || duration < 0) continue
|
||||
durations.set(path, duration)
|
||||
}
|
||||
}
|
||||
return durations
|
||||
@@ -532,10 +533,10 @@ export class CoveragePartitionCoordinator {
|
||||
private persistDurations(partitionCount: number): void {
|
||||
const reportFiles = Array.from(
|
||||
{ length: partitionCount },
|
||||
(_, index) => join(this.temporaryRoot, `partition-${index + 1}.report.json`),
|
||||
(_, index) => join(this.temporaryRoot, `partition-${index + 1}.times.json`),
|
||||
)
|
||||
const currentFiles = this.projectOf.size > 0 ? [...this.projectOf.keys()] : undefined
|
||||
writeFileDurations(this.root, collectPartitionDurations(reportFiles, this.root), currentFiles)
|
||||
writeFileDurations(this.root, collectPartitionDurations(reportFiles), currentFiles)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -584,7 +585,7 @@ export class CoveragePartitionCoordinator {
|
||||
private partitionCommand(index: number, configPath: string): CoverageCommand {
|
||||
const blobPath = join(this.blobsRoot, `partition-${index}.json`)
|
||||
const reportsDirectory = join(this.temporaryRoot, `coverage-${index}`)
|
||||
const jsonReportPath = join(this.temporaryRoot, `partition-${index}.report.json`)
|
||||
const timingReportPath = join(this.temporaryRoot, `partition-${index}.times.json`)
|
||||
const invocation = pnpmInvocation([
|
||||
'exec',
|
||||
'vitest',
|
||||
@@ -595,10 +596,10 @@ export class CoveragePartitionCoordinator {
|
||||
`--config=${this.relativePath(configPath)}`,
|
||||
'--reporter=default',
|
||||
'--reporter=blob',
|
||||
'--reporter=json',
|
||||
`--reporter=${FILE_TIMES_REPORTER}`,
|
||||
`--reporter=${CANONICAL_LOCATIONS_REPORTER}`,
|
||||
`--outputFile.blob=${this.relativePath(blobPath)}`,
|
||||
`--outputFile.json=${this.relativePath(jsonReportPath)}`,
|
||||
`--outputFile.fileTimes=${this.relativePath(timingReportPath)}`,
|
||||
`--coverage.reportsDirectory=${this.relativePath(reportsDirectory)}`,
|
||||
...this.vitestArgs,
|
||||
], { npm_execpath: this.pnpmEntrypoint })
|
||||
|
||||
@@ -24,14 +24,14 @@ describe('PR preview workflow', () => {
|
||||
expect(preview.steps.find(step => step.uses === 'actions/checkout@v6')?.with).toEqual({ 'persist-credentials': false })
|
||||
})
|
||||
|
||||
it('keeps the immutable full build and restore-only dependency cache', () => {
|
||||
it('keeps the immutable complete artifact build and restore-only dependency cache', () => {
|
||||
expect(workflow.env.PRIMARY_NODE_VERSION).toBe('24')
|
||||
expect(workflow.env.DSH_TELEMETRY_DISABLED).toBe('1')
|
||||
const commands = preview.steps.map(step => step.run)
|
||||
expect(commands).toContain('pnpm install --frozen-lockfile')
|
||||
expect(commands).toContain('pnpm run build')
|
||||
expect(commands).toContain('pnpm run build --artifacts-only')
|
||||
expect(commands).toContain('pnpm --filter @deepseek-ai/dsh-web-frontend run build:preview')
|
||||
expect(commands.indexOf('pnpm run build')).toBeLessThan(commands.indexOf('pnpm --filter @deepseek-ai/dsh-web-frontend run build:preview'))
|
||||
expect(commands.indexOf('pnpm run build --artifacts-only')).toBeLessThan(commands.indexOf('pnpm --filter @deepseek-ai/dsh-web-frontend run build:preview'))
|
||||
expect(preview.steps.filter(step => step.uses?.startsWith('actions/cache'))).toHaveLength(1)
|
||||
expect(preview.steps.find(step => step.uses === 'actions/cache/restore@v4')?.with).toMatchObject({
|
||||
key: "${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}",
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
/** Real-pnpm regression coverage for release packing, selection, hooks, and payload checks. */
|
||||
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'
|
||||
import { createServer, type ServerResponse } from 'node:http'
|
||||
import { createRequire } from 'node:module'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { officialClientBuildEnvironment, writeClientBuildRecord } from '../client-build-environment.ts'
|
||||
import { runGate, type Gate } from '../run-gates.ts'
|
||||
import { removeFixtureSafely } from '../test-fixture-cleanup.ts'
|
||||
import { releaseFamily, tarballName } from './families.ts'
|
||||
import { packedIdentity, PUBLISH_ORDER_FILE, readPublishOrder } from './tarball.ts'
|
||||
|
||||
const require = createRequire(import.meta.url)
|
||||
const packScript = resolve(import.meta.dirname, 'pack.ts')
|
||||
const tsxHook = pathToFileURL(require.resolve('tsx/esm')).href
|
||||
const pnpmEntry = join(dirname(require.resolve('pnpm')), 'bin/pnpm.cjs')
|
||||
const roots: string[] = []
|
||||
const commit = 'abcdef0'
|
||||
|
||||
function write(path: string, content: string): void {
|
||||
mkdirSync(dirname(path), { recursive: true })
|
||||
writeFileSync(path, content)
|
||||
}
|
||||
|
||||
function fixture(): string {
|
||||
const root = mkdtempSync(join(tmpdir(), 'dsh-release-pack-'))
|
||||
roots.push(root)
|
||||
write(join(root, 'package.json'), JSON.stringify({ name: '@deepseek-ai/pack-root', private: true, version: '1.2.3' }))
|
||||
write(join(root, 'pnpm-workspace.yaml'), 'packages:\n - vendor/*\n - native/*\n - packages/*/*\n - apps/*\n')
|
||||
return root
|
||||
}
|
||||
|
||||
function packageFixture(root: string, directory: string, name: string, additions: Record<string, unknown> = {}): void {
|
||||
write(join(root, directory, 'package.json'), JSON.stringify({
|
||||
name, version: '1.2.3', files: ['lib/index.js', 'hook.cjs'],
|
||||
scripts: {
|
||||
prepack: 'node hook.cjs prepack',
|
||||
prepare: 'node hook.cjs prepare',
|
||||
postpack: 'node hook.cjs postpack',
|
||||
},
|
||||
...additions,
|
||||
}))
|
||||
write(join(root, directory, 'lib/index.js'), 'export const value = 1;\n')
|
||||
write(join(root, directory, 'hook.cjs'), [
|
||||
"const fs = require('node:fs');",
|
||||
"const path = require('node:path');",
|
||||
"const manifest = require('./package.json');",
|
||||
"fs.appendFileSync(path.join(process.env.DSH_PACK_FIXTURE, 'hooks.log'), `${manifest.name}:${process.argv[2]}\\n`);",
|
||||
'',
|
||||
].join('\n'))
|
||||
}
|
||||
|
||||
function packGate(root: string, args: string[] = []): Gate {
|
||||
return {
|
||||
id: 'release-pack-fixture',
|
||||
label: 'release pack fixture',
|
||||
displayCommand: 'pnpm exec node release pack fixture',
|
||||
command: process.execPath,
|
||||
args: [pnpmEntry, '--dir', root, 'exec', process.execPath, '--import', tsxHook, packScript, '--family', 'vendor', '--out', 'packed', ...args],
|
||||
env: { npm_execpath: pnpmEntry, DSH_PACK_FIXTURE: root, DSH_CLIENT_COMMIT_HASH: commit },
|
||||
}
|
||||
}
|
||||
|
||||
async function runPack(root: string, args: string[] = []) {
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => { controller.abort() }, 90_000)
|
||||
try {
|
||||
const result = await runGate(packGate(root, args), controller.signal)
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.aborted).toBe(false)
|
||||
expect(result.signalCode).toBeNull()
|
||||
return {
|
||||
status: result.exitCode,
|
||||
stderr: result.output.filter(chunk => chunk.stream === 'stderr').map(chunk => chunk.text).join(''),
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}
|
||||
|
||||
function stopped(pid: number): boolean {
|
||||
if (process.platform === 'linux') {
|
||||
try {
|
||||
return /\)\s+Z\s/.test(readFileSync(`/proc/${pid}/stat`, 'utf8'))
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') return true
|
||||
throw error
|
||||
}
|
||||
}
|
||||
try {
|
||||
process.kill(pid, 0)
|
||||
return false
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ESRCH') return true
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function readyHookPids(root: string): number[] {
|
||||
const ready: unknown = JSON.parse(readFileSync(join(root, 'hook-ready.json'), 'utf8'))
|
||||
if (ready === null || typeof ready !== 'object' || !('hook' in ready) || !('child' in ready)
|
||||
|| typeof ready.hook !== 'number' || !Number.isSafeInteger(ready.hook) || ready.hook < 1
|
||||
|| typeof ready.child !== 'number' || !Number.isSafeInteger(ready.child) || ready.child < 1) {
|
||||
throw new Error('pack fixture hook readiness requires positive process ids')
|
||||
}
|
||||
return [ready.hook, ready.child]
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) removeFixtureSafely(root)
|
||||
})
|
||||
|
||||
describe('release pack CLI', () => {
|
||||
it.each([undefined, '1', '2'])('packs only public family members and records their publish order (concurrency=%s)', async (concurrency) => {
|
||||
const root = fixture()
|
||||
// Publication places zebra before alpha; pnpm's workspace dev edge points in the opposite direction.
|
||||
packageFixture(root, 'vendor/alpha', '@deepseek-ai/alpha', { peerDependencies: { '@deepseek-ai/zebra': '*' } })
|
||||
packageFixture(root, 'vendor/zebra', '@deepseek-ai/zebra', { devDependencies: { '@deepseek-ai/alpha': 'workspace:*' } })
|
||||
write(join(root, 'vendor/zebra/node_modules/@deepseek-ai/alpha/package.json'), JSON.stringify({ name: '@deepseek-ai/alpha', version: '1.2.3' }))
|
||||
packageFixture(root, 'vendor/private', '@deepseek-ai/private', { private: true })
|
||||
packageFixture(root, 'native/unselected', '@deepseek-ai/unselected')
|
||||
const family = releaseFamily('vendor')
|
||||
const members = family.publishOrder(family.members(root)).order
|
||||
const expectedOrder = members.map(tarballName)
|
||||
|
||||
const result = await runPack(root, concurrency === undefined ? [] : ['--concurrency', concurrency])
|
||||
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
expect(readPublishOrder(join(root, 'packed'))).toEqual(expectedOrder)
|
||||
expect(readdirSync(join(root, 'packed')).sort()).toEqual([...expectedOrder, PUBLISH_ORDER_FILE].sort())
|
||||
for (const member of members) {
|
||||
expect(packedIdentity(join(root, 'packed', tarballName(member))))
|
||||
.toEqual({ name: member.name, version: member.version })
|
||||
}
|
||||
const hooks = readFileSync(join(root, 'hooks.log'), 'utf8').trim().split('\n')
|
||||
const expectedHooks = members.flatMap(member => ['prepack', 'prepare', 'postpack'].map(phase => `${member.name}:${phase}`))
|
||||
expect([...hooks].sort()).toEqual([...expectedHooks].sort())
|
||||
if (concurrency !== '2') expect(hooks).toEqual(expectedHooks)
|
||||
}, 120_000)
|
||||
|
||||
it('overlaps two independent packs and keeps later packages within the concurrency limit', async () => {
|
||||
const root = fixture()
|
||||
const names = ['alpha', 'bravo', 'charlie'].map(name => `@deepseek-ai/${name}`)
|
||||
for (const name of names) {
|
||||
const directory = `vendor/${name.slice('@deepseek-ai/'.length)}`
|
||||
packageFixture(root, directory, name, {
|
||||
scripts: { prepack: 'node hook.cjs start', postpack: 'node hook.cjs finish' },
|
||||
})
|
||||
write(join(root, directory, 'hook.cjs'), [
|
||||
"const { get } = require('node:http');",
|
||||
"const { name } = require('./package.json');",
|
||||
'const url = `${process.env.DSH_PACK_COORDINATOR}/${process.argv[2]}?name=${encodeURIComponent(name)}&pid=${process.pid}`;',
|
||||
'const request = get(url, { agent: false }, response => {',
|
||||
' response.resume();',
|
||||
" response.on('end', () => { if (response.statusCode !== 200) process.exitCode = 1; });",
|
||||
'});',
|
||||
"request.on('error', error => { console.error(error); process.exitCode = 1; });",
|
||||
'',
|
||||
].join('\n'))
|
||||
}
|
||||
|
||||
const active = new Set<string>()
|
||||
const waiting = new Map<string, ServerResponse>()
|
||||
const started: string[] = []
|
||||
const completed: string[] = []
|
||||
const pids = new Set<number>()
|
||||
const errors: string[] = []
|
||||
let maximumActive = 0
|
||||
const server = createServer((request, response) => {
|
||||
const url = new URL(request.url ?? '/', 'http://localhost')
|
||||
const name = url.searchParams.get('name') ?? ''
|
||||
const pid = Number(url.searchParams.get('pid'))
|
||||
if (!names.includes(name) || !Number.isSafeInteger(pid) || pid < 1
|
||||
|| (url.pathname !== '/start' && url.pathname !== '/finish')) {
|
||||
errors.push(`unexpected hook request ${request.url ?? ''}`)
|
||||
response.writeHead(400).end()
|
||||
return
|
||||
}
|
||||
pids.add(pid)
|
||||
if (url.pathname === '/start') {
|
||||
if (active.has(name)) errors.push(`${name} started twice`)
|
||||
started.push(name)
|
||||
active.add(name)
|
||||
maximumActive = Math.max(maximumActive, active.size)
|
||||
waiting.set(name, response)
|
||||
} else {
|
||||
if (!active.delete(name)) errors.push(`${name} finished without starting`)
|
||||
completed.push(name)
|
||||
response.end()
|
||||
}
|
||||
})
|
||||
const release = (name: string | undefined): void => {
|
||||
const response = name === undefined ? undefined : waiting.get(name)
|
||||
if (name === undefined || response === undefined) throw new Error('pack hook has not reached its barrier')
|
||||
waiting.delete(name)
|
||||
response.end()
|
||||
}
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => { controller.abort() }, 90_000)
|
||||
let pending: ReturnType<typeof runGate> | undefined
|
||||
try {
|
||||
await new Promise<void>((resolveListen, rejectListen) => {
|
||||
server.once('error', rejectListen)
|
||||
server.listen(0, '127.0.0.1', () => {
|
||||
server.off('error', rejectListen)
|
||||
resolveListen()
|
||||
})
|
||||
})
|
||||
const address = server.address()
|
||||
if (address === null || typeof address === 'string') throw new Error('pack coordinator has no TCP address')
|
||||
const gate = packGate(root, ['--concurrency', '2'])
|
||||
gate.env = { ...gate.env, DSH_PACK_COORDINATOR: `http://127.0.0.1:${address.port}` }
|
||||
pending = runGate(gate, controller.signal)
|
||||
|
||||
await vi.waitFor(() => { expect(started).toHaveLength(2) }, { timeout: 30_000, interval: 25 })
|
||||
expect(active.size).toBe(2)
|
||||
expect([...pids].every(pid => !stopped(pid))).toBe(true)
|
||||
expect(completed).toEqual([])
|
||||
release(started[0])
|
||||
|
||||
await vi.waitFor(() => { expect(started).toHaveLength(3) }, { timeout: 30_000, interval: 25 })
|
||||
expect(completed).toEqual([started[0]])
|
||||
expect(active.size).toBe(2)
|
||||
expect(maximumActive).toBe(2)
|
||||
release(started[1])
|
||||
release(started[2])
|
||||
|
||||
const result = await pending
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.aborted).toBe(false)
|
||||
expect(result.signalCode).toBeNull()
|
||||
expect(result.exitCode, result.output.map(chunk => chunk.text).join('')).toBe(0)
|
||||
expect(errors).toEqual([])
|
||||
expect([...started].sort()).toEqual(names)
|
||||
expect([...completed].sort()).toEqual(names)
|
||||
expect(active.size).toBe(0)
|
||||
expect(maximumActive).toBe(2)
|
||||
await vi.waitFor(() => { expect([...pids].every(stopped)).toBe(true) }, { timeout: 10_000, interval: 25 })
|
||||
expect(readPublishOrder(join(root, 'packed'))).toHaveLength(3)
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
controller.abort()
|
||||
try {
|
||||
await pending
|
||||
} finally {
|
||||
if (server.listening) {
|
||||
const closed = new Promise<void>((resolveClose, rejectClose) => {
|
||||
server.close((error) => {
|
||||
if (error === undefined) resolveClose()
|
||||
else rejectClose(error)
|
||||
})
|
||||
})
|
||||
server.closeAllConnections()
|
||||
await closed
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 120_000)
|
||||
|
||||
it('does not broaden an empty parallel family to the whole workspace', async () => {
|
||||
const root = fixture()
|
||||
packageFixture(root, 'vendor/private', '@deepseek-ai/private', { private: true })
|
||||
packageFixture(root, 'native/unselected', '@deepseek-ai/unselected')
|
||||
|
||||
const result = await runPack(root, ['--concurrency', '8'])
|
||||
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
expect(readFileSync(join(root, 'packed', PUBLISH_ORDER_FILE), 'utf8')).toBe('\n')
|
||||
expect(readdirSync(join(root, 'packed'))).toEqual([PUBLISH_ORDER_FILE])
|
||||
expect(existsSync(join(root, 'hooks.log'))).toBe(false)
|
||||
})
|
||||
|
||||
it('leaves no publish order after a pack hook fails', async () => {
|
||||
const root = fixture()
|
||||
packageFixture(root, 'vendor/failing', '@deepseek-ai/failing', { scripts: { prepack: 'node failing.cjs' } })
|
||||
write(join(root, 'vendor/failing/failing.cjs'), "throw new Error('pack fixture failure')\n")
|
||||
|
||||
const result = await runPack(root, ['--concurrency', '8'])
|
||||
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('pack fixture failure')
|
||||
expect(existsSync(join(root, 'packed', PUBLISH_ORDER_FILE))).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects a successful command that did not produce the expected member tarball', async () => {
|
||||
const root = fixture()
|
||||
packageFixture(root, 'vendor/expected', '@deepseek-ai/expected', { scripts: { prepack: 'node rename.cjs' } })
|
||||
write(join(root, 'vendor/expected/rename.cjs'), [
|
||||
"const fs = require('node:fs');",
|
||||
"const manifest = require('./package.json');",
|
||||
"manifest.name = '@deepseek-ai/renamed';",
|
||||
"fs.writeFileSync('package.json', JSON.stringify(manifest));",
|
||||
'',
|
||||
].join('\n'))
|
||||
|
||||
const result = await runPack(root, ['--concurrency', '8'])
|
||||
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('@deepseek-ai/expected produced no tarball')
|
||||
expect(existsSync(join(root, 'packed', 'deepseek-ai-renamed-1.2.3.tgz'))).toBe(true)
|
||||
expect(existsSync(join(root, 'packed', PUBLISH_ORDER_FILE))).toBe(false)
|
||||
})
|
||||
|
||||
it('validates every parallel tarball before recording the publish order', async () => {
|
||||
const root = fixture()
|
||||
packageFixture(root, 'packages/util/alpha', '@deepseek-ai/alpha')
|
||||
packageFixture(root, 'packages/util/zebra', '@deepseek-ai/zebra', { files: ['lib/index.js', 'src/index.ts'] })
|
||||
write(join(root, 'packages/util/zebra/src/index.ts'), 'export const source = 1\n')
|
||||
write(join(root, 'apps/web/dist/index.html'), '<main></main>')
|
||||
writeClientBuildRecord(root, officialClientBuildEnvironment(root, { DSH_CLIENT_COMMIT_HASH: commit }))
|
||||
|
||||
const result = await runPack(root, ['--family', 'dsh', '--concurrency', '8'])
|
||||
|
||||
expect(result.status).not.toBe(0)
|
||||
expect(result.stderr).toContain('@deepseek-ai/zebra publishes source file package/src/index.ts')
|
||||
expect(readdirSync(join(root, 'packed')).filter(name => name.endsWith('.tgz'))).toHaveLength(2)
|
||||
expect(existsSync(join(root, 'packed', PUBLISH_ORDER_FILE))).toBe(false)
|
||||
})
|
||||
|
||||
it('stops the ready pack hook and its stalled descendant before fixture cleanup', async () => {
|
||||
const root = fixture()
|
||||
packageFixture(root, 'vendor/stalled', '@deepseek-ai/stalled', { scripts: { prepack: 'node stalled.cjs' } })
|
||||
write(join(root, 'vendor/stalled/stalled.cjs'), [
|
||||
"const { spawn } = require('node:child_process');",
|
||||
"spawn(process.execPath, ['descendant.cjs'], { stdio: 'inherit' });",
|
||||
'',
|
||||
].join('\n'))
|
||||
write(join(root, 'vendor/stalled/descendant.cjs'), [
|
||||
"const fs = require('node:fs');",
|
||||
"const path = require('node:path');",
|
||||
"process.on('SIGTERM', () => {});",
|
||||
'setInterval(() => {}, 1000);',
|
||||
"fs.writeFileSync(path.join(process.env.DSH_PACK_FIXTURE, 'hook-ready.json'), JSON.stringify({ hook: process.ppid, child: process.pid }));",
|
||||
'',
|
||||
].join('\n'))
|
||||
const controller = new AbortController()
|
||||
const timer = setTimeout(() => { controller.abort() }, 90_000)
|
||||
const pending = runGate(packGate(root, ['--concurrency', '8']), controller.signal)
|
||||
try {
|
||||
let pids: number[] = []
|
||||
await vi.waitFor(() => {
|
||||
pids = readyHookPids(root)
|
||||
expect(pids.every(pid => !stopped(pid))).toBe(true)
|
||||
}, { timeout: 30_000, interval: 25 })
|
||||
|
||||
controller.abort()
|
||||
const result = await pending
|
||||
|
||||
expect(result.aborted).toBe(true)
|
||||
expect(result.status).toBe('failed')
|
||||
await vi.waitFor(() => { expect(pids.every(stopped)).toBe(true) }, { timeout: 10_000, interval: 25 })
|
||||
expect(existsSync(join(root, 'packed', PUBLISH_ORDER_FILE))).toBe(false)
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
controller.abort()
|
||||
await pending
|
||||
}
|
||||
}, 120_000)
|
||||
})
|
||||
+38
-21
@@ -1,6 +1,7 @@
|
||||
/**
|
||||
* Pack one release family's whole publish set into a single directory, in
|
||||
* publish order, and record that order for the publish step.
|
||||
* Pack one release family's whole publish set into a single directory and
|
||||
* record its publish order for the publish step. Default serial packs execute
|
||||
* in that order; explicitly parallel packs use pnpm's bounded scheduler.
|
||||
*
|
||||
* The pack step is the release boundary: it runs without credentials, produces
|
||||
* every tarball from one commit, and hands the publish step exactly those bytes
|
||||
@@ -19,16 +20,13 @@ import { PUBLISH_ORDER_FILE, tarballFiles } from './tarball.ts'
|
||||
const DEFAULT_OUTPUT = 'dist/npm'
|
||||
|
||||
/**
|
||||
* Pack one member and check what its tarball carries.
|
||||
* Check one member's expected tarball and its published files.
|
||||
* @param family - the release family being packed.
|
||||
* @param member - the member to pack.
|
||||
* @param destination - absolute output directory.
|
||||
* @returns The tarball filename.
|
||||
*/
|
||||
async function packMember(family: ReleaseFamily, member: ReleaseMember, destination: string): Promise<string> {
|
||||
const invocation = pnpmInvocation(['--dir', member.directory, 'pack', '--pack-destination', destination])
|
||||
await runConcurrent(invocation.command, invocation.args)
|
||||
|
||||
function validatePackedMember(family: ReleaseFamily, member: ReleaseMember, destination: string): string {
|
||||
const filename = tarballName(member)
|
||||
const tarball = join(destination, filename)
|
||||
if (!existsSync(tarball)) throw new Error(`${member.name} produced no tarball at ${tarball}`)
|
||||
@@ -36,6 +34,38 @@ async function packMember(family: ReleaseFamily, member: ReleaseMember, destinat
|
||||
return filename
|
||||
}
|
||||
|
||||
/**
|
||||
* Pack serial members in publish order, or batch an explicitly parallel family.
|
||||
* @param family - Release family whose payload checks apply to every tarball.
|
||||
* @param members - Exact public members in the recorded publish order.
|
||||
* @param destination - Absolute output directory.
|
||||
* @param concurrency - Maximum active package packs; one preserves serial publication order.
|
||||
* @returns Validated tarball filenames in publish order.
|
||||
*/
|
||||
async function packMembers(
|
||||
family: ReleaseFamily, members: readonly ReleaseMember[], destination: string, concurrency: number,
|
||||
): Promise<string[]> {
|
||||
if (concurrency === 1) {
|
||||
const order: string[] = []
|
||||
for (const member of members) {
|
||||
const invocation = pnpmInvocation(['--dir', member.directory, 'pack', '--pack-destination', destination])
|
||||
await runConcurrent(invocation.command, invocation.args)
|
||||
order.push(validatePackedMember(family, member, destination))
|
||||
}
|
||||
return order
|
||||
}
|
||||
if (members.length === 0) return []
|
||||
const invocation = pnpmInvocation([
|
||||
'pack',
|
||||
'--recursive',
|
||||
`--workspace-concurrency=${String(concurrency)}`,
|
||||
...members.map(member => `--filter=${member.name}`),
|
||||
'--pack-destination', destination,
|
||||
])
|
||||
await runConcurrent(invocation.command, invocation.args)
|
||||
return members.map(member => validatePackedMember(family, member, destination))
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns The validated `--concurrency` value; 1 (the default) packs the
|
||||
* members one at a time, exactly as the credentialed publish workflows run it.
|
||||
@@ -68,20 +98,7 @@ async function main(): Promise<void> {
|
||||
rmSync(destination, { recursive: true, force: true })
|
||||
mkdirSync(destination, { recursive: true })
|
||||
|
||||
// Members pack in a bounded pool; the recorded publish order stays the
|
||||
// members' order regardless of completion order, because each worker writes
|
||||
// its result at the member's own position.
|
||||
const order = new Array<string>(members.length)
|
||||
let cursor = 0
|
||||
await Promise.all(Array.from({ length: Math.min(concurrency, members.length) }, async () => {
|
||||
while (cursor < members.length) {
|
||||
const index = cursor
|
||||
cursor += 1
|
||||
const member = members[index]
|
||||
if (member === undefined) break
|
||||
order[index] = await packMember(family, member, destination)
|
||||
}
|
||||
}))
|
||||
const order = await packMembers(family, members, destination, concurrency)
|
||||
writeFileSync(join(destination, PUBLISH_ORDER_FILE), `${order.join('\n')}\n`)
|
||||
|
||||
console.log(`release pack: family ${family.id}, ${String(order.length)} tarball(s) in ${values.out ?? DEFAULT_OUTPUT}`)
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
/** Build benchmark artifacts with TypeScript diagnostics owned by required full builds. */
|
||||
|
||||
import { resolve } from 'node:path'
|
||||
import { buildLibraryArtifacts, runPnpmCommand } from './compile-referenced-projects.ts'
|
||||
|
||||
/**
|
||||
* Build native support, libraries, benchmark workers, and Web assets before running the existing benchmark suite.
|
||||
* @param root - Repository root containing all build configurations.
|
||||
* @throws If any build or the benchmark suite fails.
|
||||
*/
|
||||
export function runCiBench(root: string): void {
|
||||
runPnpmCommand(root, ['run', 'build:native-system'])
|
||||
buildLibraryArtifacts(root, false, 'benchmark-emit')
|
||||
runPnpmCommand(root, ['exec', 'tsdown', '--config-loader', 'native', '--config', 'benchmarks/tsdown.config.ts'])
|
||||
runPnpmCommand(root, ['run', 'build:web'])
|
||||
runPnpmCommand(root, ['run', 'test:bench:built'])
|
||||
}
|
||||
|
||||
if (import.meta.main) runCiBench(resolve(import.meta.dirname, '..'))
|
||||
@@ -156,6 +156,13 @@ function withPlatform<T>(platform: NodeJS.Platform, action: () => T): T {
|
||||
try { return action() } finally { Object.defineProperty(process, 'platform', original) }
|
||||
}
|
||||
|
||||
// Mutates worker-global state: only use for synchronous, non-concurrent graph inspection.
|
||||
function withNodeVersion<T>(version: string, action: () => T): T {
|
||||
const original = Object.getOwnPropertyDescriptor(process.versions, 'node')!
|
||||
Object.defineProperty(process.versions, 'node', { ...original, value: version })
|
||||
try { return action() } finally { Object.defineProperty(process.versions, 'node', original) }
|
||||
}
|
||||
|
||||
describe('CI worker allocation', () => {
|
||||
it.each([1, 2, 4, 8, 16, 64])('shares a %i CPU coverage budget without multiplying pools', (cpus) => {
|
||||
const env = ciWorkerEnvironment('ci-coverage', {}, cpus)
|
||||
@@ -243,7 +250,7 @@ describe('gate graph validation', () => {
|
||||
await expect(runGates(subject, subject.length, execute)).resolves.toHaveLength(subject.length)
|
||||
})
|
||||
|
||||
it('builds the native addon before benchmarks through the ci-bench script chain', () => {
|
||||
it('uses the internal CI benchmark runner without changing the public benchmark scripts', () => {
|
||||
const subject = withPnpmEntrypoint(() => gatesForMode('ci-bench'))
|
||||
const { scripts } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as {
|
||||
scripts: Record<string, string>
|
||||
@@ -253,8 +260,8 @@ describe('gate graph validation', () => {
|
||||
expect(subject).toHaveLength(1)
|
||||
expect(subject[0]).toMatchObject({
|
||||
id: 'bench',
|
||||
displayCommand: 'pnpm run test:bench',
|
||||
args: ['/private/pnpm.cjs', 'run', 'test:bench'],
|
||||
displayCommand: 'pnpm exec tsx scripts/run-ci-bench.ts',
|
||||
args: ['/private/pnpm.cjs', 'exec', 'tsx', 'scripts/run-ci-bench.ts'],
|
||||
})
|
||||
expect(scripts['test:bench']).toBe('npm run build:bench && npm run build:web && npm run test:bench:built')
|
||||
expect(scripts['build:bench']).toBe(
|
||||
@@ -814,20 +821,76 @@ describe('Typert contract preparation', () => {
|
||||
})
|
||||
|
||||
describe('Node compatibility graph', () => {
|
||||
it('runs the jsdom environment smoke on every advertised Node line', () => {
|
||||
const subject = withPnpmEntrypoint(() => gatesForMode('node-compat'))
|
||||
it.each(['22.19.0', '24.9.0', '26.0.0'])('runs all source compatibility smokes serially on Node %s', (version) => {
|
||||
const subject = withNodeVersion(version, () =>
|
||||
withPnpmEntrypoint(() => gatesForMode('node-compat')))
|
||||
|
||||
expect(subject.find(item => item.id === 'vitest-jsdom-smoke')).toMatchObject({
|
||||
label: 'Vitest jsdom smoke',
|
||||
expect(subject.filter(item => item.id === 'source-compat-smokes')).toMatchObject([{
|
||||
label: 'source compatibility smokes',
|
||||
args: [
|
||||
'/private/pnpm.cjs',
|
||||
'exec',
|
||||
'vitest',
|
||||
'run',
|
||||
'packages/workflow/workflow-ptc/tests/source-runtime.compat.spec.ts',
|
||||
'packages/session/session-persistence-jsonl/tests/zstd.compat.spec.ts',
|
||||
'apps/cli/tests/source-launch.compat.spec.ts',
|
||||
'scripts/vitest-environment.compat.spec.ts',
|
||||
'packages/boot/app-boot/tests/profile-resolution.spec.ts',
|
||||
'packages/boot/app-boot/tests/profile-resolution-service.spec.ts',
|
||||
'packages/boot/app-boot/tests/profile-resolution-worker-bootstrap.spec.ts',
|
||||
'--no-file-parallelism',
|
||||
'--maxWorkers=1',
|
||||
],
|
||||
env: { VITEST_MAX_WORKERS: '1' },
|
||||
}])
|
||||
})
|
||||
|
||||
it.each(['1', '8'])('keeps source compatibility smokes serial with inherited VITEST_MAX_WORKERS=%s', (workers) => {
|
||||
const subject = withEnv('VITEST_MAX_WORKERS', workers, () =>
|
||||
withPnpmEntrypoint(() => gatesForMode('node-compat')))
|
||||
|
||||
expect(subject.find(item => item.id === 'source-compat-smokes')?.env).toEqual({ VITEST_MAX_WORKERS: '1' })
|
||||
})
|
||||
|
||||
it.each([undefined, '1'])('builds the complete Node 22 artifacts once with skip-typecheck=%s', (skipTypecheck) => {
|
||||
const subject = withNodeVersion('22.19.0', () =>
|
||||
withEnv('DSH_NODE_COMPAT_SKIP_TYPECHECK', skipTypecheck, () =>
|
||||
withPnpmEntrypoint(() => gatesForMode('node-compat'))))
|
||||
|
||||
expect(subject.filter(item => item.displayCommand === 'pnpm run build --artifacts-only')).toHaveLength(1)
|
||||
expect(subject.find(item => item.id === 'build')?.args).toEqual([
|
||||
'/private/pnpm.cjs', 'run', 'build', '--artifacts-only',
|
||||
])
|
||||
expect(subject.map(item => item.id)).not.toContain('build:web')
|
||||
expect(subject.find(item => item.id === 'cli-lazy-search-startup-smoke')).toMatchObject({
|
||||
needs: ['build'],
|
||||
env: { DSH_REQUIRE_BUILT_CLI_SMOKE: '1' },
|
||||
})
|
||||
})
|
||||
|
||||
it.each(['passed', 'failed'] as const)('runs the Node 22 lazy-search smoke only after a passed build (%s)', async (buildStatus) => {
|
||||
const subject = withNodeVersion('22.19.0', () =>
|
||||
withEnv('DSH_NODE_COMPAT_SKIP_TYPECHECK', '1', () =>
|
||||
withPnpmEntrypoint(() => gatesForMode('node-compat'))))
|
||||
let buildCompleted = false
|
||||
let smokeStarted = false
|
||||
const results = await runGates(subject, 2, async (item) => {
|
||||
if (item.id === 'build') {
|
||||
buildCompleted = true
|
||||
return resultFor(item, buildStatus)
|
||||
}
|
||||
if (item.id === 'cli-lazy-search-startup-smoke') {
|
||||
expect(buildCompleted).toBe(true)
|
||||
smokeStarted = true
|
||||
}
|
||||
return resultFor(item)
|
||||
})
|
||||
|
||||
expect(smokeStarted).toBe(buildStatus === 'passed')
|
||||
expect(results.find(item => item.gate.id === 'cli-lazy-search-startup-smoke')?.status)
|
||||
.toBe(buildStatus === 'passed' ? 'passed' : 'skipped')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Node 24 lane ownership', () => {
|
||||
|
||||
+10
-24
@@ -285,7 +285,7 @@ export function gatesForMode(selected: Mode): Gate[] {
|
||||
case 'ci-unit':
|
||||
return ciUnitGates()
|
||||
case 'ci-bench':
|
||||
return [pnpmScript('bench', 'test:bench', { label: 'performance benchmarks' })]
|
||||
return [pnpmExec('bench', ['tsx', 'scripts/run-ci-bench.ts'], { label: 'performance benchmarks' })]
|
||||
case 'ci-snapshot':
|
||||
return [ciBuildGate(), snapshotGate()]
|
||||
case 'ci-artifacts':
|
||||
@@ -409,45 +409,31 @@ function nodeCompatGates(): Gate[] {
|
||||
return [
|
||||
...typecheck,
|
||||
pnpmScript('build', 'build', {
|
||||
...pnpmInvocation(['run', 'build', '--artifacts-only']),
|
||||
displayCommand: 'pnpm run build --artifacts-only',
|
||||
...typecheck.length === 0 ? {} : { needs: ['typecheck'] },
|
||||
}),
|
||||
pnpmScript('build:web', 'build:web', {
|
||||
label: 'Web frontend build',
|
||||
needs: ['build'],
|
||||
}),
|
||||
...nodeCompatSmokeGates({ cliSmoke: true }),
|
||||
]
|
||||
}
|
||||
|
||||
function nodeCompatSmokeGates(options: { cliSmoke?: boolean } = {}): Gate[] {
|
||||
const gates: Gate[] = [
|
||||
pnpmExec('source-worker-smoke', [
|
||||
// Serial files keep CLI and PTC children from overlapping. Vitest's
|
||||
// environment worker override takes precedence over its CLI flags.
|
||||
pnpmExec('source-compat-smokes', [
|
||||
'vitest',
|
||||
'run',
|
||||
'packages/workflow/workflow-ptc/tests/source-runtime.compat.spec.ts',
|
||||
], { label: 'source worker smoke' }),
|
||||
pnpmExec('jsonl-zstd-smoke', [
|
||||
'vitest',
|
||||
'run',
|
||||
'packages/session/session-persistence-jsonl/tests/zstd.compat.spec.ts',
|
||||
], { label: 'JSONL Zstandard smoke' }),
|
||||
pnpmExec('dsh-source-launch-smoke', [
|
||||
'vitest',
|
||||
'run',
|
||||
'apps/cli/tests/source-launch.compat.spec.ts',
|
||||
], { label: 'dsh source-launch smoke' }),
|
||||
pnpmExec('vitest-jsdom-smoke', [
|
||||
'vitest',
|
||||
'run',
|
||||
'scripts/vitest-environment.compat.spec.ts',
|
||||
], { label: 'Vitest jsdom smoke' }),
|
||||
pnpmExec('profile-resolution-smoke', [
|
||||
'vitest',
|
||||
'run',
|
||||
'packages/boot/app-boot/tests/profile-resolution.spec.ts',
|
||||
'packages/boot/app-boot/tests/profile-resolution-service.spec.ts',
|
||||
'packages/boot/app-boot/tests/profile-resolution-worker-bootstrap.spec.ts',
|
||||
], { label: 'profile resolution smoke' }),
|
||||
'--no-file-parallelism',
|
||||
'--maxWorkers=1',
|
||||
], { label: 'source compatibility smokes', env: { VITEST_MAX_WORKERS: '1' } }),
|
||||
]
|
||||
if (options.cliSmoke) {
|
||||
gates.push(
|
||||
@@ -458,7 +444,7 @@ function nodeCompatSmokeGates(options: { cliSmoke?: boolean } = {}): Gate[] {
|
||||
], {
|
||||
label: 'CLI lazy-search startup smoke',
|
||||
env: { DSH_REQUIRE_BUILT_CLI_SMOKE: '1' },
|
||||
needs: ['build:web'],
|
||||
needs: ['build'],
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -146,7 +146,9 @@ for (const [file, jobIds] of [['release.yml', ['dependencies', 'pack']], ['relea
|
||||
const output = family === 'dsh' ? 'dist/npm' : 'dist/npm-vendor'
|
||||
expect(job.steps[0]?.with?.['fetch-depth']).toBe(0)
|
||||
expect(commands).toContain('pnpm run release:verify --family ' + family)
|
||||
expect(commands).toContain('pnpm run ' + (family === 'dsh' ? 'build:official' : 'build:lib:host'))
|
||||
expect(commands).toContain(family === 'dsh'
|
||||
? "${{ github.event_name == 'pull_request' && 'pnpm run build --artifacts-only --profile official' || 'pnpm run build:official' }}"
|
||||
: "${{ github.event_name == 'pull_request' && 'pnpm exec tsx scripts/compile-referenced-projects.ts host-libraries' || 'pnpm run build:lib:host' }}")
|
||||
expect(commands).toContain('pnpm run release:pack --family ' + family + ' --out ' + output + ' --concurrency 8')
|
||||
expect(commands).toContain('pnpm run release:verify-packed-install --family ' + family + ' --from ' + output
|
||||
+ (family === 'dsh' ? ' --from dist/npm-vendor --from dist/npm-landlock' : ''))
|
||||
|
||||
@@ -4,6 +4,7 @@ import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { parseMarkdown } from './markdown.ts'
|
||||
import {
|
||||
normalizeTranslationMarkdownLinks,
|
||||
rewriteTranslationLinkLocales,
|
||||
@@ -183,6 +184,33 @@ describe('translation link locale validation', () => {
|
||||
})
|
||||
|
||||
describe('translation link rewriting and normalization', () => {
|
||||
it('reuses an authored tree without changing its offsets, switcher exemption, or reference resolution', () => {
|
||||
const root = fixture()
|
||||
const input = '# 指南\n\n[English](guide.md) | 中文\n\n[行内](reference.md?view=full&x=1#overview "标题")\n\n[引用][ref]\n\n[ref]: <reference.md#tail> "标注"\n[ref]: unpaired.md\n'
|
||||
const context = linkContext(root, 'docs/guide.zh.md')
|
||||
const tree = parseMarkdown(input)
|
||||
const originalTree = structuredClone(tree)
|
||||
|
||||
expect(normalizeTranslationMarkdownLinks(input, context, ['guide.md'], tree)).toBe(
|
||||
'# 指南\n\n[English](guide.md) | 中文\n\n[行内](dsh-translation-target:docs/reference.md?view=full&x=1#overview "标题")\n\n[引用][ref]\n\n[ref]: <dsh-translation-target:docs/reference.md#tail> "标注"\n[ref]: unpaired.md\n',
|
||||
)
|
||||
expect(translationLinkLocaleViolations(input, context, ['guide.md'], tree)).toEqual([
|
||||
{
|
||||
sourcePath: 'docs/guide.zh.md',
|
||||
line: 5,
|
||||
url: 'reference.md?view=full&x=1#overview',
|
||||
expectedUrl: 'reference.zh.md?view=full&x=1#overview',
|
||||
},
|
||||
{
|
||||
sourcePath: 'docs/guide.zh.md',
|
||||
line: 9,
|
||||
url: 'reference.md#tail',
|
||||
expectedUrl: 'reference.zh.md#tail',
|
||||
},
|
||||
])
|
||||
expect(tree).toEqual(originalTree)
|
||||
})
|
||||
|
||||
it('rewrites only the destination while preserving the suffix and title', () => {
|
||||
const root = fixture()
|
||||
const input = '[概览](reference.md?view=full&mode=all#overview "reference.md title")\n'
|
||||
|
||||
@@ -229,8 +229,8 @@ function visitDocumentLinkNodes(
|
||||
markdown: string,
|
||||
skipTargets: readonly string[],
|
||||
visitor: (node: LinkNode) => void,
|
||||
tree: Nodes = parseMarkdown(markdown),
|
||||
): void {
|
||||
const tree = parseMarkdown(markdown)
|
||||
const switcherOffset = languageSwitcherLinkOffset(tree, markdown, skipTargets)
|
||||
const referencedIdentifiers = new Set<string>()
|
||||
const visitedDefinitions = new Set<string>()
|
||||
@@ -255,20 +255,29 @@ function visitResolvedDocumentLinks(
|
||||
context: TranslationLinkContext,
|
||||
skipTargets: readonly string[],
|
||||
visitor: (node: LinkNode, destination: MarkdownDestination, resolved: ResolvedTranslationLink) => void,
|
||||
tree?: Nodes,
|
||||
): void {
|
||||
visitDocumentLinkNodes(markdown, skipTargets, (node) => {
|
||||
if (isExternalOrAbsoluteMarkdownUrl(node.url)) return
|
||||
const destination = markdownDestination(markdown, node)
|
||||
const resolved = resolveTranslationLink(node.url, context, destination.url)
|
||||
if (resolved !== undefined) visitor(node, destination, resolved)
|
||||
})
|
||||
}, tree)
|
||||
}
|
||||
|
||||
/** Return one violation per wrong-locale link or link definition. */
|
||||
/**
|
||||
* Return one violation per wrong-locale link or link definition.
|
||||
* @param markdown - Exact authored Markdown text used for link offsets.
|
||||
* @param context - Repository and locale used to resolve relative links.
|
||||
* @param skipTargets - Accepted language-switcher destinations.
|
||||
* @param tree - Optional GFM parse of this exact Markdown text; never a normalized or edited document.
|
||||
* @returns Wrong-locale links in document order.
|
||||
*/
|
||||
export function translationLinkLocaleViolations(
|
||||
markdown: string,
|
||||
context: TranslationLinkContext,
|
||||
skipTargets: readonly string[] = [],
|
||||
tree?: Nodes,
|
||||
): TranslationLinkLocaleViolation[] {
|
||||
const violations: TranslationLinkLocaleViolation[] = []
|
||||
visitResolvedDocumentLinks(markdown, context, skipTargets, (node, destination, resolved) => {
|
||||
@@ -279,7 +288,7 @@ export function translationLinkLocaleViolations(
|
||||
url: destination.url,
|
||||
expectedUrl: resolved.expectedUrl,
|
||||
})
|
||||
})
|
||||
}, tree)
|
||||
return violations
|
||||
}
|
||||
|
||||
@@ -297,11 +306,19 @@ export function rewriteTranslationLinkLocales(
|
||||
return { content: applyReplacements(markdown, replacements), rewritten: replacements.length }
|
||||
}
|
||||
|
||||
/** Normalize only paired-document locale paths while retaining every other byte and URL suffix. */
|
||||
/**
|
||||
* Normalize only paired-document locale paths while retaining every other byte and URL suffix.
|
||||
* @param markdown - Exact authored Markdown text used for link offsets.
|
||||
* @param context - Repository and locale used to resolve relative links.
|
||||
* @param skipTargets - Accepted language-switcher destinations.
|
||||
* @param tree - Optional GFM parse of this exact Markdown text; never a normalized or edited document.
|
||||
* @returns Markdown with paired-document destinations replaced by semantic targets.
|
||||
*/
|
||||
export function normalizeTranslationMarkdownLinks(
|
||||
markdown: string,
|
||||
context: TranslationLinkContext,
|
||||
skipTargets: readonly string[] = [],
|
||||
tree?: Nodes,
|
||||
): string {
|
||||
const replacements: Replacement[] = []
|
||||
visitResolvedDocumentLinks(markdown, context, skipTargets, (_node, destination, resolved) => {
|
||||
@@ -309,7 +326,7 @@ export function normalizeTranslationMarkdownLinks(
|
||||
destination,
|
||||
`dsh-translation-target:${resolved.pair.source}${resolved.suffix}`,
|
||||
))
|
||||
})
|
||||
}, tree)
|
||||
return applyReplacements(markdown, replacements)
|
||||
}
|
||||
|
||||
|
||||
@@ -68,11 +68,10 @@ function headingText(node: Nodes): string {
|
||||
* Fenced code blocks and generated regions, which the pairing gate requires to
|
||||
* be identical on both sides, are left out of each section's blocks.
|
||||
*/
|
||||
function sections(markdown: string): Section[] {
|
||||
function sections(markdown: string, tree: Nodes = parseTranslationMarkdown(markdown)): Section[] {
|
||||
const frontmatter = /^---\n[\s\S]*?\n---(?:\n|$)/.exec(markdown)?.[0].length ?? 0
|
||||
const result: Section[] = [{ blocks: frontmatter > 0 ? [markdown.slice(0, frontmatter)] : [] }]
|
||||
const regionLines = generatedRegions(markdown).map(region => [region.begin + 1, region.end + 1] as const)
|
||||
const tree = parseTranslationMarkdown(markdown)
|
||||
if (tree.type !== 'root') throw new Error('Markdown parser returned a non-root tree')
|
||||
for (const node of tree.children) {
|
||||
if ((node.position?.start.offset ?? 0) < frontmatter) continue
|
||||
@@ -106,6 +105,7 @@ function sectionHash(blocks: string[]): string {
|
||||
* @param en - English document text.
|
||||
* @param zh - Chinese document text.
|
||||
* @param context - Link-resolution inputs for both sides.
|
||||
* @param trees - Optional GFM parses of the exact English and Chinese inputs; normalized text is parsed separately when it changes.
|
||||
* @returns Entries in document order.
|
||||
* @throws Error when the two sides have different heading counts or malformed generated regions.
|
||||
*/
|
||||
@@ -114,9 +114,12 @@ export function computeTranslationPairingRecord(
|
||||
en: string,
|
||||
zh: string,
|
||||
context: TranslationPairingRecordContext,
|
||||
trees?: { readonly en: Nodes; readonly zh: Nodes },
|
||||
): TranslationPairingRecord {
|
||||
const enSections = sections(normalizeTranslationMarkdownLinks(en, { ...context, sourcePath: paths.source }))
|
||||
const zhSections = sections(normalizeTranslationMarkdownLinks(zh, { ...context, sourcePath: paths.zh }))
|
||||
const normalizedEn = normalizeTranslationMarkdownLinks(en, { ...context, sourcePath: paths.source }, [], trees?.en)
|
||||
const normalizedZh = normalizeTranslationMarkdownLinks(zh, { ...context, sourcePath: paths.zh }, [], trees?.zh)
|
||||
const enSections = sections(normalizedEn, normalizedEn === en ? trees?.en : undefined)
|
||||
const zhSections = sections(normalizedZh, normalizedZh === zh ? trees?.zh : undefined)
|
||||
if (enSections.length !== zhSections.length) {
|
||||
throw new Error(`${paths.source} has ${enSections.length - 1} heading(s) but ${paths.zh} has ${zhSections.length - 1}`)
|
||||
}
|
||||
|
||||
@@ -250,6 +250,31 @@ describe('translation pairing records', () => {
|
||||
.toEqual(record(`${en}\nSee [bar](bar.zh.md).\n`, `${zh}\nSee [bar](bar.md).\n`))
|
||||
})
|
||||
|
||||
it('retains section hashes after normalized links move later heading offsets', () => {
|
||||
const english = '# Guide\n\n[Reference](bar.md?x=1&y=2#tail)\n\n## Tail\n\nTail body.\n'
|
||||
const chinese = '# 指南\n\n[参考](bar.zh.md?x=1&y=2#tail)\n\n## 尾部\n\n尾部正文。\n'
|
||||
const trees = { en: parseTranslationMarkdown(english), zh: parseTranslationMarkdown(chinese) }
|
||||
const originalTrees = structuredClone(trees)
|
||||
const computed = computeTranslationPairingRecord(paths, english, chinese, context, trees)
|
||||
|
||||
expect([...computed]).toEqual([
|
||||
['/guide', { en: '3f43f06d1244d874', zh: '021cfe53b05ba084' }],
|
||||
['/guide/tail', { en: '945d8537bf1f2b5f', zh: '0a851cb395705d76' }],
|
||||
])
|
||||
expect(computed).toEqual(record(english, chinese))
|
||||
expect(trees).toEqual(originalTrees)
|
||||
})
|
||||
|
||||
it('retains section hashes when the authored tree needs no link normalization', () => {
|
||||
const english = '# Guide\n\nBody.\n'
|
||||
const chinese = '# 指南\n\n正文。\n'
|
||||
const trees = { en: parseTranslationMarkdown(english), zh: parseTranslationMarkdown(chinese) }
|
||||
|
||||
expect([...computeTranslationPairingRecord(paths, english, chinese, context, trees)]).toEqual([
|
||||
['/guide', { en: '5e1bfa97a822af7a', zh: '3409c92bef7306bb' }],
|
||||
])
|
||||
})
|
||||
|
||||
it('refuses sides with different heading counts', () => {
|
||||
expect(() => record(en, `${zh}\n## 额外\n`)).toThrow('docs/foo.md has 3 heading(s) but docs/foo.zh.md has 4')
|
||||
})
|
||||
|
||||
@@ -239,8 +239,12 @@ for (const source of [...pairAnchors].sort()) {
|
||||
const sourceText = sourceContent.toString('utf8')
|
||||
const zhText = zhContent.toString('utf8')
|
||||
let current: ReturnType<typeof computeTranslationPairingRecord>
|
||||
let sourceTree: ReturnType<typeof parseTranslationMarkdown>
|
||||
let zhTree: ReturnType<typeof parseTranslationMarkdown>
|
||||
try {
|
||||
current = computeTranslationPairingRecord(paths, sourceText, zhText, recordContext)
|
||||
sourceTree = parseTranslationMarkdown(sourceText)
|
||||
zhTree = parseTranslationMarkdown(zhText)
|
||||
current = computeTranslationPairingRecord(paths, sourceText, zhText, recordContext, { en: sourceTree, zh: zhTree })
|
||||
} catch (error) {
|
||||
errors.push(`${source} ↔ ${zh}: ${error instanceof Error ? error.message : String(error)}`)
|
||||
state.set(source, 'out-of-sync')
|
||||
@@ -265,13 +269,13 @@ for (const source of [...pairAnchors].sort()) {
|
||||
sourcePath: source,
|
||||
isTranslationPairSource,
|
||||
repositoryFileExists,
|
||||
}, zhSwitcherTargets),
|
||||
}, zhSwitcherTargets, sourceTree),
|
||||
...translationLinkLocaleViolations(zhText, {
|
||||
repoRoot: root,
|
||||
sourcePath: zh,
|
||||
isTranslationPairSource,
|
||||
repositoryFileExists,
|
||||
}, sourceSwitcherTargets),
|
||||
}, sourceSwitcherTargets, zhTree),
|
||||
]) {
|
||||
errors.push(`${violation.sourcePath}:${violation.line}: link target ${JSON.stringify(violation.url)} uses the wrong locale; expected ${JSON.stringify(violation.expectedUrl)}`)
|
||||
state.set(source, 'out-of-sync')
|
||||
@@ -309,8 +313,6 @@ for (const source of [...pairAnchors].sort()) {
|
||||
state.set(source, 'out-of-sync')
|
||||
}
|
||||
|
||||
const sourceTree = parseTranslationMarkdown(sourceText)
|
||||
const zhTree = parseTranslationMarkdown(zhText)
|
||||
if (!hasLanguageSwitcher(zhTree, zhText, sourceSwitcherTargets)) {
|
||||
errors.push(`${zh}: missing language switcher — no link to ${basename(source)}`)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user