perf(windows): bound parallel PE header discovery
This commit is contained in:
1 parent
d148052e83
commit
e7a9e2683d
6 files changed
+174
-11
No files matched your search
@@ -41,7 +41,7 @@ export interface WindowsCodeSigningOptions {
|
||||
/**
|
||||
* Enumerate PE files by content without following links.
|
||||
* @param root Materialized directory to scan.
|
||||
* @returns Sorted PE paths; rejects malformed Windows executables and links.
|
||||
* @returns Sorted PE paths; eight bounded readers reject links or malformed executables and drain open handles before failure.
|
||||
*/
|
||||
export function windowsRuntimeCode(root: string): Promise<string[]>
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { promisify } from 'node:util'
|
||||
import { scrubWindowsSigningEnvironment } from './windows-sign.mjs'
|
||||
import { recordPackagingEvent } from './packaging-run.mjs'
|
||||
import { inspectSignaturesBatched } from './windows-signature-batch.mjs'
|
||||
import { packagingStep } from './packaging-step.mjs'
|
||||
|
||||
/**
|
||||
* Read Windows trust, timestamp and signer identity using the engine's bundled modules, without accessing the private key.
|
||||
@@ -52,17 +53,32 @@ export async function preserveWindowsRuntimeSignature(path, options) {
|
||||
/**
|
||||
* Enumerate Windows code without following links or treating foreign .node files as PE binaries.
|
||||
* @param {string} root - Owned, materialized runtime directory.
|
||||
* @returns {Promise<string[]>} Sorted real PE files; rejects links and malformed Windows executable files.
|
||||
* @returns {Promise<string[]>} Sorted real PE files; eight bounded readers reject links or malformed executables and drain open handles before failure.
|
||||
*/
|
||||
export async function windowsRuntimeCode(root) {
|
||||
const rootStat = await lstat(root)
|
||||
if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error('Windows code: expected a real directory')
|
||||
return packagingStep(process.env.DSH_DESKTOP_PACKAGING_RUN_DIR, 'windows:enumerate-code', async () => {
|
||||
const candidates = []
|
||||
async function collect(directory) {
|
||||
const rootStat = await lstat(directory)
|
||||
if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error('Windows code: expected a real directory')
|
||||
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
||||
const path = join(directory, entry.name)
|
||||
if (entry.isSymbolicLink()) throw new Error(`Windows code: directory links are not signable: ${path}`)
|
||||
if (entry.isDirectory()) await collect(path)
|
||||
else if (entry.isFile()) candidates.push(path)
|
||||
}
|
||||
}
|
||||
await collect(root)
|
||||
return inspectPortableExecutables(candidates)
|
||||
})
|
||||
}
|
||||
|
||||
/** Close every active file before returning a sorted inventory or the first inspection failure. */
|
||||
async function inspectPortableExecutables(candidates) {
|
||||
const files = []
|
||||
for (const entry of await readdir(root, { withFileTypes: true })) {
|
||||
const path = join(root, entry.name)
|
||||
if (entry.isSymbolicLink()) throw new Error(`Windows code: directory links are not signable: ${path}`)
|
||||
if (entry.isDirectory()) { files.push(...await windowsRuntimeCode(path)); continue }
|
||||
if (!entry.isFile()) continue
|
||||
let next = 0
|
||||
let failure
|
||||
async function inspect(path) {
|
||||
const file = await open(path, 'r')
|
||||
let portableExecutable = false
|
||||
let windowsCandidate = false
|
||||
@@ -80,11 +96,26 @@ export async function windowsRuntimeCode(root) {
|
||||
if (portableExecutable) files.push(path)
|
||||
else if (windowsCandidate || ['.exe', '.dll', '.pyd'].includes(extname(path).toLowerCase())) throw new Error(`Windows code: invalid PE file: ${path}`)
|
||||
}
|
||||
async function worker() {
|
||||
while (failure === undefined && next < candidates.length) {
|
||||
const path = candidates[next++]
|
||||
try { await inspect(path) }
|
||||
catch (error) { failure ??= { error } }
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(8, candidates.length) }, worker))
|
||||
if (failure !== undefined) throw failure.error
|
||||
return files.sort()
|
||||
}
|
||||
|
||||
/** Drain each public-key verification batch before signing or reporting a failure. */
|
||||
async function inspectWindowsCode(files, inspect) {
|
||||
return packagingStep(process.env.DSH_DESKTOP_PACKAGING_RUN_DIR, 'windows:inspect-signatures',
|
||||
() => inspectWindowsCodeBatch(files, inspect))
|
||||
}
|
||||
|
||||
/** Complete bounded inspections without allowing callers to proceed past an active failed batch. */
|
||||
async function inspectWindowsCodeBatch(files, inspect) {
|
||||
if (inspect === inspectWindowsRuntimeSignature) return inspectSignaturesBatched(files)
|
||||
const signatures = []
|
||||
for (let offset = 0; offset < files.length; offset += 4) {
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
/** PE discovery keeps content checks and drains bounded file reads before failure. */
|
||||
import { mkdir, mkdtemp, rm, writeFile, type FileHandle } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { setImmediate } from 'node:timers/promises'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { windowsRuntimeCode } from '../scripts/windows-runtime-signature.mjs'
|
||||
|
||||
const hooks = vi.hoisted(() => ({
|
||||
opened: undefined as ((handle: FileHandle) => void) | undefined,
|
||||
}))
|
||||
|
||||
vi.mock('node:fs/promises', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('node:fs/promises')>()
|
||||
return {
|
||||
...actual,
|
||||
open: async (...args: Parameters<typeof actual.open>) => {
|
||||
const handle = await actual.open(...args)
|
||||
hooks.opened?.(handle)
|
||||
return handle
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
const roots: string[] = []
|
||||
|
||||
async function fixture(entries: Record<string, string | Buffer>): Promise<string> {
|
||||
const root = await mkdtemp(join(tmpdir(), 'windows-runtime-code-'))
|
||||
roots.push(root)
|
||||
for (const [name, contents] of Object.entries(entries)) {
|
||||
const path = join(root, name)
|
||||
await mkdir(dirname(path), { recursive: true })
|
||||
await writeFile(path, contents)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
function executable(): Buffer {
|
||||
const bytes = Buffer.alloc(128)
|
||||
bytes.writeUInt16LE(0x5a4d, 0)
|
||||
bytes.writeUInt32LE(64, 0x3c)
|
||||
bytes.writeUInt32LE(0x4550, 64)
|
||||
return bytes
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
hooks.opened = undefined
|
||||
vi.restoreAllMocks()
|
||||
for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('discovers extensionless PE files in nested directories and excludes foreign native modules', async () => {
|
||||
const root = await fixture({
|
||||
'nested/deeper/extensionless': executable(),
|
||||
'nested/runtime.node': executable(),
|
||||
'a.exe': executable(),
|
||||
'nested/deeper/foreign.node': Buffer.from([0x7f, 0x45, 0x4c, 0x46]),
|
||||
'readme.txt': 'ordinary data',
|
||||
})
|
||||
expect(await windowsRuntimeCode(root)).toEqual([
|
||||
'nested/deeper/extensionless', 'nested/runtime.node', 'a.exe',
|
||||
].map(name => join(root, name)).sort())
|
||||
})
|
||||
|
||||
it.each([
|
||||
['broken.exe', Buffer.from('not PE')],
|
||||
['nested/extensionless', Buffer.from('MZ')],
|
||||
['nested/broken.node', Buffer.from('MZ')],
|
||||
['nested/deeper/broken.dll', Buffer.alloc(64)],
|
||||
])('rejects malformed Windows code at %s', async (name, contents) => {
|
||||
const root = await fixture({ [name]: contents })
|
||||
await expect(windowsRuntimeCode(root)).rejects.toThrow(`Windows code: invalid PE file: ${join(root, name)}`)
|
||||
})
|
||||
|
||||
it('limits open files to eight and drains pending closes before rejecting a read failure', async () => {
|
||||
const root = await fixture(Object.fromEntries(Array.from({ length: 24 }, (_, index) => [
|
||||
`${String(index).padStart(2, '0')}.txt`, 'ordinary data',
|
||||
])))
|
||||
const releaseReads = Promise.withResolvers<undefined>()
|
||||
const releaseCloses = Promise.withResolvers<undefined>()
|
||||
const failedClosed = Promise.withResolvers<undefined>()
|
||||
const failure = new Error('controlled PE header read failure')
|
||||
let opened = 0
|
||||
let active = 0
|
||||
let peak = 0
|
||||
let settled = false
|
||||
hooks.opened = (handle) => {
|
||||
const index = opened++
|
||||
active++
|
||||
peak = Math.max(peak, active)
|
||||
const close = handle.close.bind(handle)
|
||||
vi.spyOn(handle, 'read').mockImplementationOnce(async () => {
|
||||
await releaseReads.promise
|
||||
if (index === 0) throw failure
|
||||
return { bytesRead: 0, buffer: Buffer.alloc(64) }
|
||||
})
|
||||
vi.spyOn(handle, 'close').mockImplementationOnce(async () => {
|
||||
if (index !== 0) await releaseCloses.promise
|
||||
await close()
|
||||
active--
|
||||
if (index === 0) failedClosed.resolve(undefined)
|
||||
})
|
||||
}
|
||||
const operation = windowsRuntimeCode(root).then(
|
||||
(result) => { settled = true; return result },
|
||||
(error: unknown) => { settled = true; return error },
|
||||
)
|
||||
try {
|
||||
await vi.waitFor(() => { expect(opened).toBe(8) })
|
||||
expect(active).toBe(8)
|
||||
releaseReads.resolve(undefined)
|
||||
await failedClosed.promise
|
||||
await setImmediate()
|
||||
expect(settled).toBe(false)
|
||||
expect(opened).toBe(8)
|
||||
expect(active).toBe(7)
|
||||
releaseCloses.resolve(undefined)
|
||||
expect(await operation).toBe(failure)
|
||||
expect(opened).toBe(8)
|
||||
expect(peak).toBe(8)
|
||||
expect(active).toBe(0)
|
||||
} finally {
|
||||
hooks.opened = undefined
|
||||
releaseReads.resolve(undefined)
|
||||
releaseCloses.resolve(undefined)
|
||||
await operation
|
||||
}
|
||||
})
|
||||
@@ -33,8 +33,8 @@
|
||||
en: 5401b605a2f3a44e
|
||||
zh: fc1f6ee667bcd20d
|
||||
/windows-signing-and-runner-operations/signature-cache-and-concurrent-stages:
|
||||
en: 1ab28ad6c6489102
|
||||
zh: 0ee6df9aaaeddace
|
||||
en: 78593443275fe259
|
||||
zh: 9a8c1b541f479273
|
||||
/windows-signing-and-runner-operations/verification-limits:
|
||||
en: 94a471410c84a1c7
|
||||
zh: 115228e4ca36581d
|
||||
|
||||
@@ -90,6 +90,8 @@ Preflight, primary-runtime signing, application-runtime signing and artifact cre
|
||||
|
||||
Cache-hit copying, hashing and per-file candidate trust checks run with `DSH_DESKTOP_WINDOWS_SIGNATURE_CACHE_CONCURRENCY` workers from `.env.windows` (default `4`, integer `1`–`8`). After all restores finish, bounded PowerShell batches verify every restored target's trust, timestamp and certificate before cache misses enter serial hardware signing. A restore failure stops new restores and drains active workers; a target-verification failure drains its inspection batch and prevents hardware signing. Runtime smoke checks and final integrity validation remain required.
|
||||
|
||||
PE discovery reads every regular file's header with at most eight open files, rejects links and malformed Windows executables, and drains active reads and closes before reporting failure. Packaging records `windows:enumerate-code` and `windows:inspect-signatures` as nested phases of signing and artifact verification; their timings are included in the enclosing stages.
|
||||
|
||||
Windows packaging reads `DSH_DESKTOP_WINDOWS_PACK_CONCURRENCY` from `.env.windows` or the selected preset: a positive integer, defaulting to `4`. Packaging batches first-party and vendor tarballs with bounded workers, then validates every payload and records publication order. Setting the limit to `1` selects serial packing; standalone `release:pack` defaults to `1`.
|
||||
|
||||
Each runtime stage writes `SIGNATURE_CACHE_SUMMARY` to stdout and its packaging journal, including the actual directory, policy identity, hit/miss counts, newly published and retained entries, signing requests, avoided signing requests and validation failures. Timings distinguish signing, trust verification of restored and newly signed files, and restoration; restoration includes its trust check and staging cleanup. These counters exclude preflight, final artifact signing and the outer runtime signature checks. Stage lock events record waiting separately. Concurrent timings sum per-file work rather than stage wall time.
|
||||
|
||||
@@ -90,6 +90,8 @@ Windows 打包命令通过 `DESKTOP_PACKAGING_RECORD` 输出 `.desktop-build/pac
|
||||
|
||||
缓存命中的复制、摘要和逐文件候选信任检查使用 `.env.windows` 中的 `DSH_DESKTOP_WINDOWS_SIGNATURE_CACHE_CONCURRENCY` 个工作任务(默认 `4`,整数 `1`–`8`)。所有恢复完成后,有界 PowerShell 批次逐项验证恢复目标的信任、时间戳和证书,通过后未命中项才进入串行硬件签名。恢复失败会停止派发新的恢复任务并等待在途工作结束;目标验签失败会等待当前检查批次结束并阻止硬件签名。运行时 smoke 检查和最终完整性校验仍须执行。
|
||||
|
||||
PE 发现会读取每个普通文件的头部,同时最多打开八个文件,拒绝链接和损坏的 Windows 可执行文件,并在报告失败前等待在途读取和关闭结束。打包将 `windows:enumerate-code` 与 `windows:inspect-signatures` 记录为签名和产物验证的嵌套阶段;其耗时已包含在外层阶段中。
|
||||
|
||||
Windows 打包从 `.env.windows` 或所选预设读取 `DSH_DESKTOP_WINDOWS_PACK_CONCURRENCY`:必须为正整数,默认 `4`。Windows 使用有界 worker 批量打包第一方和 vendor tarball,随后校验每个产物并记录发布顺序。设为 `1` 时串行打包;独立的 `release:pack` 默认使用 `1`。
|
||||
|
||||
每个运行时阶段向标准输出及打包日志写入 `SIGNATURE_CACHE_SUMMARY`,包含实际目录、策略标识、命中及未命中数、新发布及保留项数、签名请求数、省去的签名请求数和验证失败数。计时区分签名、恢复文件和新签名文件的信任验证及恢复;恢复耗时包含其信任检查和暂存清理。这些统计不包含预检、最终产物签名及外层运行时验签。阶段锁事件单独记录等待时间。 并发计时累加逐文件工作耗时,不代表阶段墙钟时间。
|
||||
|
||||
Reference in new issue
Block a user