diff --git a/benchmarks/AGENTS.md b/benchmarks/AGENTS.md index 2f40d70afb..0754dff1da 100644 --- a/benchmarks/AGENTS.md +++ b/benchmarks/AGENTS.md @@ -14,3 +14,4 @@ This tree owns required, repository-level performance gates whose measured user - Exercise production entry points. Do not copy product algorithms, add production exports solely for measurement, or turn benchmark completion into duplicate semantic assertions. - A compiled worker may bundle a private integration adapter when no public Node export exposes the measured user path. Keep package imports external so product services resolve through their built package exports. - Record the workload, timing boundary, memory endpoint, calibration reference, alternatives, and known exclusions in the owning Agent Note. +- `pnpm run test:bench:memory 4g|8g` reruns the files listed in [run-memory-pressure-bench.ts](../scripts/run-memory-pressure-bench.ts) inside a swapless cgroup capped at 4 or 8 GiB, with unchanged budgets. Keep a file's budgets valid under both caps when it joins that list. diff --git a/package.json b/package.json index 5fe3f6cd9d..9f20bb9734 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "test:e2e": "vitest run --config vitest.e2e.config.ts", "test:bench": "npm run build:bench && npm run build:web && npm run test:bench:built", "test:bench:built": "vitest run --config vitest.bench.config.ts", + "test:bench:memory": "tsx scripts/run-memory-pressure-bench.ts", "test:expected": "vitest run --config vitest.expected.config.ts", "test:expected:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.expected.config.ts", "test:approval-policy": "node --test .github/review-ownership/check-approval.test.mjs .github/review-ownership/blame-ownership.test.mjs .github/review-ownership/author-weight.test.mjs", diff --git a/scripts/run-ci-bench.ts b/scripts/run-ci-bench.ts index 8d2e4c042a..c4778982e1 100644 --- a/scripts/run-ci-bench.ts +++ b/scripts/run-ci-bench.ts @@ -4,15 +4,24 @@ import { resolve } from 'node:path' import { buildLibraryArtifacts, runPnpmCommand } from './compile-referenced-projects.ts' /** - * Build native support, libraries, benchmark workers, and Web assets before running the existing benchmark suite. + * Build native support, libraries, benchmark workers, and Web assets that benchmark files consume. * @param root - Repository root containing all build configurations. - * @throws If any build or the benchmark suite fails. + * @throws If any build fails. */ -export function runCiBench(root: string): void { +export function buildCiBenchArtifacts(root: string): void { runPnpmCommand(root, ['run', 'build:native-system']) buildLibraryArtifacts(root, false, 'benchmark-emit') runPnpmCommand(root, ['exec', 'tsdown', '--config-loader', 'native', '--config', 'benchmarks/tsdown.config.ts']) runPnpmCommand(root, ['run', 'build:web']) +} + +/** + * Build benchmark artifacts, then run the existing benchmark suite. + * @param root - Repository root containing all build configurations. + * @throws If any build or the benchmark suite fails. + */ +export function runCiBench(root: string): void { + buildCiBenchArtifacts(root) runPnpmCommand(root, ['run', 'test:bench:built']) } diff --git a/scripts/run-memory-pressure-bench.spec.ts b/scripts/run-memory-pressure-bench.spec.ts new file mode 100644 index 0000000000..96bd3f1d0e --- /dev/null +++ b/scripts/run-memory-pressure-bench.spec.ts @@ -0,0 +1,29 @@ +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + MEMORY_PRESSURE_FILES, + memoryPressureMachine, + memoryScopeCommand, +} from './run-memory-pressure-bench.ts' + +const root = resolve(import.meta.dirname, '..') + +describe('memory-pressure benchmark launcher', () => { + it('selects existing benchmark files', () => { + for (const file of MEMORY_PRESSURE_FILES) expect(existsSync(resolve(root, file)), file).toBe(true) + }) + + it('accepts only the modeled machines', () => { + expect(memoryPressureMachine('4g')).toBe('4g') + expect(memoryPressureMachine('8g')).toBe('8g') + expect(() => memoryPressureMachine('16g')).toThrow('usage') + expect(() => memoryPressureMachine(undefined)).toThrow('usage') + }) + + it('confines the command to the machine memory without swap', () => { + expect(memoryScopeCommand('4g', ['node', 'x.js'])).toEqual([ + 'systemd-run', '--user', '--scope', '--quiet', '-p', 'MemoryMax=4G', '-p', 'MemorySwapMax=0', '--', 'node', 'x.js', + ]) + }) +}) diff --git a/scripts/run-memory-pressure-bench.ts b/scripts/run-memory-pressure-bench.ts new file mode 100644 index 0000000000..2b33e09259 --- /dev/null +++ b/scripts/run-memory-pressure-bench.ts @@ -0,0 +1,119 @@ +/** Run the memory-sensitive benchmark files inside a Linux cgroup that models a 4 GB or 8 GB machine. */ + +import { spawnSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { join, resolve } from 'node:path' +import { getHeapStatistics } from 'node:v8' +import { buildCiBenchArtifacts } from './run-ci-bench.ts' +import { pnpmCommand } from './release/process.ts' + +/** Modeled machines: memory available to the whole benchmark process tree, without swap. */ +export const MEMORY_PRESSURE_MACHINES = { '4g': 4, '8g': 8 } as const + +/** Modeled machine name. */ +export type MemoryPressureMachine = keyof typeof MEMORY_PRESSURE_MACHINES + +/** + * Benchmark files that run under the limit. They cover the Host operations with the largest measured + * resident memory, the longest Session open, and the Chromium long-history workflow with its Host. + */ +export const MEMORY_PRESSURE_FILES = [ + 'benchmarks/session-corpus/session-corpus.bench.ts', + 'benchmarks/session-corpus/projection-list.bench.ts', + 'benchmarks/session-open/session-open.bench.ts', + 'benchmarks/long-session-browser/long-session.bench.ts', +] as const + +const USAGE = `usage: run-memory-pressure-bench.ts <${Object.keys(MEMORY_PRESSURE_MACHINES).join('|')}>` + +/** + * Parse a modeled machine name. + * @param value - Command-line machine name. + * @returns The machine name. + * @throws If the name is not a modeled machine. + */ +export function memoryPressureMachine(value: string | undefined): MemoryPressureMachine { + if (value !== undefined && Object.hasOwn(MEMORY_PRESSURE_MACHINES, value)) return value as MemoryPressureMachine + throw new Error(USAGE) +} + +/** + * Build the systemd transient-scope command that confines `command` to the machine's memory. + * @param machine - Modeled machine. + * @param command - Command and arguments to run inside the scope. + * @returns The `systemd-run` argument vector. + */ +export function memoryScopeCommand(machine: MemoryPressureMachine, command: readonly string[]): string[] { + return [ + 'systemd-run', '--user', '--scope', '--quiet', + '-p', `MemoryMax=${String(MEMORY_PRESSURE_MACHINES[machine])}G`, + '-p', 'MemorySwapMax=0', + '--', ...command, + ] +} + +function run(root: string, command: readonly string[]): number { + const [file, ...args] = command + if (file === undefined) throw new Error('run-memory-pressure-bench: empty command') + const result = spawnSync(file, args, { cwd: root, stdio: 'inherit' }) + if (result.error !== undefined) throw result.error + return result.status ?? 1 +} + +function cgroupFile(name: string): string { + const line = readFileSync('/proc/self/cgroup', 'utf8').split('\n').find(entry => entry.startsWith('0::')) + if (line === undefined) throw new Error('run-memory-pressure-bench: the process is not in a cgroup v2 hierarchy') + return readFileSync(join('/sys/fs/cgroup', line.slice(3), name), 'utf8').trim() +} + +/** + * Run the selected benchmark files inside the current scope and report the scope's memory facts. + * @param root - Repository root with built benchmark artifacts. + * @param machine - Modeled machine that the current scope must enforce. + * @returns The benchmark exit status. + * @throws If the current cgroup does not enforce the machine's memory. + */ +export function runScopedMemoryPressureBench(root: string, machine: MemoryPressureMachine): number { + const limitBytes = MEMORY_PRESSURE_MACHINES[machine] * 1024 ** 3 + if (Number(cgroupFile('memory.max')) !== limitBytes || process.constrainedMemory() !== limitBytes) { + throw new Error(`run-memory-pressure-bench: the scope does not enforce ${machine}: memory.max=${cgroupFile('memory.max')}`) + } + const status = run(root, [...pnpmCommand(), 'exec', 'vitest', 'run', '--config', 'vitest.bench.config.ts', ...MEMORY_PRESSURE_FILES]) + const oomKills = /^oom_kill (\d+)$/m.exec(cgroupFile('memory.events'))?.[1] + console.log(JSON.stringify({ + benchmark: `memory-pressure/${machine}`, + limitMb: limitBytes / 1024 ** 2, + swapLimit: cgroupFile('memory.swap.max'), + nodeHeapLimitMb: Math.round(getHeapStatistics().heap_size_limit / 1024 ** 2), + peakMb: Math.round(Number(cgroupFile('memory.peak')) / 1024 ** 2), + oomKills: Number(oomKills), + files: MEMORY_PRESSURE_FILES, + status, + })) + return status +} + +/** + * Build benchmark artifacts unconfined, then rerun this script inside a transient memory-limited scope. + * @param root - Repository root. + * @param machine - Modeled machine. + * @returns The scoped benchmark exit status. + * @throws If the host is not Linux or a build fails. + */ +export function runMemoryPressureBench(root: string, machine: MemoryPressureMachine): number { + if (process.platform !== 'linux') { + throw new Error('run-memory-pressure-bench: memory limits need Linux cgroup v2 and a systemd user manager') + } + buildCiBenchArtifacts(root) + return run(root, memoryScopeCommand(machine, [ + ...pnpmCommand(), 'exec', 'tsx', 'scripts/run-memory-pressure-bench.ts', '--scoped', machine, + ])) +} + +if (import.meta.main) { + const root = resolve(import.meta.dirname, '..') + const args = process.argv.slice(2) + process.exitCode = args[0] === '--scoped' + ? runScopedMemoryPressureBench(root, memoryPressureMachine(args[1])) + : runMemoryPressureBench(root, memoryPressureMachine(args[0])) +}