test(fs): benchmark glob through Native executor
This commit is contained in:
1 parent
ef284800b9
commit
7253e123bd
4 files changed
+117
No files matched your search
@@ -44,6 +44,9 @@
|
||||
/deepseek-ai-dsh-tool-fs-search/understand-the-implementation/how-a-search-runs:
|
||||
en: 0c33819a6261ffaf
|
||||
zh: 99845a1621f4cc12
|
||||
/deepseek-ai-dsh-tool-fs-search/understand-the-implementation/native-mode-benchmark:
|
||||
en: 031654f59e63d07e
|
||||
zh: a3b488fb1880d2f5
|
||||
/deepseek-ai-dsh-tool-fs-search/understand-the-implementation/two-budgets-two-artifacts:
|
||||
en: b7e8c2191f8227dc
|
||||
zh: a064e2e03761bfd5
|
||||
|
||||
@@ -109,6 +109,20 @@ Without an explicit `path`, `glob` narrows traversal for patterns such as `packa
|
||||
|
||||
Each call resolves the packaged binary (`@vscode/ripgrep`, or the executable's `-rg` sidecar in a pkg single-file runtime), prepends `--no-config` so a host `RIPGREP_CONFIG_PATH` cannot inject a `--pre` preprocessor into the unconfined spawn, and passes every model-controlled value as a plain argv element — no shell layer exists, so no quoting applies. Collect-mode budgets bound complete stdout and a stderr tail; a lossy stdout read fails as `SEARCH_RAW_OUTPUT_OVERFLOW` rather than parsing a silently-partial stream. The tools never read a raw spill path.
|
||||
|
||||
### Native-mode benchmark
|
||||
|
||||
After `pnpm run build`, compile the package-local diagnostic with external runtime dependencies, then run it under plain Node:
|
||||
|
||||
```sh
|
||||
node --input-type=module <<'JS'
|
||||
import { build } from 'tsdown';
|
||||
await build({ config: false, tsconfig: false, dts: false, entry: 'packages/fs/tool-fs-search/tests/glob-native.perf.ts', outDir: 'packages/fs/tool-fs-search/tests/.dsh-build', clean: false, deps: { neverBundle: [/^[^./]/] } });
|
||||
JS
|
||||
node packages/fs/tool-fs-search/tests/.dsh-build/glob-native.perf.mjs
|
||||
```
|
||||
|
||||
The diagnostic creates 20,000 unrelated files and six matches, then measures five `ctx.tools.execute()` calls through the Native executor with a real Agent, working-directory service, and local subprocess backend. It reports raw timings, the median, and a hash of the complete value, text, and presentation metadata. `--plugin <built-module>` selects another search-package revision; `--workspace <directory>` opts into a read-only existing workspace instead of the synthetic tree. Setup, model generation, UI, and Session persistence are excluded; there is no timing threshold.
|
||||
|
||||
### Two budgets, two artifacts
|
||||
|
||||
Raw stdout and stderr are internal transport details; the tools always collect the complete result in memory, and only the inline page is capped. When a call yields more logical results than the inline cap, a best-effort spill saves the complete formatted preview to the spill store and the page carries its locator, while dispatches whose full value never enters model context skip the spill. Missing or failed spill keeps the inline page and reports that the complete result could not be saved — never an error. The collection and spill handoff live in `src/search-core.ts` and `src/presentation.ts`.
|
||||
|
||||
@@ -109,6 +109,20 @@ Node 部署在受支持的 macOS、Linux 与 Windows 目标上获得 `@vscode/ri
|
||||
|
||||
每次调用解析打包二进制(`@vscode/ripgrep`,或 pkg 单文件运行时中可执行程序的 `-rg` 伴随文件),前置 `--no-config`,使宿主的 `RIPGREP_CONFIG_PATH` 无法向不受约束的 spawn 注入 `--pre` 预处理器,并把每个模型控制的值作为普通 argv 元素传入——不存在 shell 层,因此不涉及 shell 引号处理。collect 模式预算限制完整 stdout 与 stderr 尾部;lossy stdout 读取以 `SEARCH_RAW_OUTPUT_OVERFLOW` 失败,而不是解析静默不完整的流。工具从不读取原始 spill 路径。
|
||||
|
||||
### Native 模式 benchmark
|
||||
|
||||
运行 `pnpm run build` 后,将包内诊断脚本编译为外部引用运行时依赖的产物,再用普通 Node 运行:
|
||||
|
||||
```sh
|
||||
node --input-type=module <<'JS'
|
||||
import { build } from 'tsdown';
|
||||
await build({ config: false, tsconfig: false, dts: false, entry: 'packages/fs/tool-fs-search/tests/glob-native.perf.ts', outDir: 'packages/fs/tool-fs-search/tests/.dsh-build', clean: false, deps: { neverBundle: [/^[^./]/] } });
|
||||
JS
|
||||
node packages/fs/tool-fs-search/tests/.dsh-build/glob-native.perf.mjs
|
||||
```
|
||||
|
||||
诊断脚本创建 20,000 个无关文件和六个匹配文件,然后通过 Native executor 测量五次 `ctx.tools.execute()` 调用,使用真实 Agent、工作目录服务和本地 subprocess 后端。输出包括原始耗时、中位数,以及完整值、文本和展示元数据的哈希。`--plugin <built-module>` 选择其他搜索包版本;`--workspace <directory>` 显式选择只读的现有工作区,替代合成目录。计时排除初始化、模型生成、UI 和 Session 持久化;不设耗时阈值。
|
||||
|
||||
### 两类预算、两类产物
|
||||
|
||||
原始 stdout 与 stderr 是内部传输细节;工具始终把完整结果收集到内存中,只有内联页面设有上限。当调用产生超过内联上限的逻辑结果时,尽力而为的 spill 会把完整格式化预览保存到 spill 存储,页面携带其 locator;完整值不会进入模型上下文的分派则跳过 spill。spill 缺失或失败时保留内联页面,并报告完整结果无法保存——绝不会成为错误。收集与 spill 交接位于 `src/search-core.ts` 与 `src/presentation.ts`。
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Threshold-free Native glob diagnostic over built runtime packages. The default
|
||||
* workload has 20,000 unrelated files and six matches. An explicit --workspace
|
||||
* selects a read-only real-workspace measurement; --plugin selects a built search
|
||||
* package revision. Setup, model generation, UI, and Session persistence are not timed.
|
||||
*/
|
||||
|
||||
import assert from 'node:assert/strict'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises'
|
||||
import { cpus, tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { parseArgs } from 'node:util'
|
||||
import { Context } from '@deepseek-ai/cordis'
|
||||
import { mountAgentLoopTestDependencies, mountAgentLoopTestHarness } from '@deepseek-ai/dsh-agent-loop-testkit'
|
||||
import { ToolCallId } from '@deepseek-ai/dsh-llm'
|
||||
import { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import LocalSubprocessRuntime from '@deepseek-ai/dsh-subprocess-local'
|
||||
import * as ToolFsSearch from '@deepseek-ai/dsh-tool-fs-search'
|
||||
|
||||
const TARGET = 'packages/core/agent-loop/src'
|
||||
const SAMPLES = 5
|
||||
const DEPENDENCIES = 1_000
|
||||
const FILES_PER_DEPENDENCY = 20
|
||||
const MATCHES = 6
|
||||
|
||||
async function measure(): Promise<void> {
|
||||
const { values } = parseArgs({ options: { plugin: { type: 'string' }, workspace: { type: 'string' } } })
|
||||
const search = values.plugin === undefined
|
||||
? ToolFsSearch
|
||||
: await import(pathToFileURL(resolve(values.plugin)).href) as typeof ToolFsSearch
|
||||
const temporary = values.workspace === undefined
|
||||
const root = values.workspace === undefined ? await mkdtemp(join(tmpdir(), 'dsh-native-glob-perf-')) : resolve(values.workspace)
|
||||
const ctx = new Context()
|
||||
try {
|
||||
if (temporary) {
|
||||
await mkdir(join(root, TARGET), { recursive: true })
|
||||
for (let index = 0; index < MATCHES; index++) {
|
||||
const file = join(root, TARGET, `file-${index}.ts`)
|
||||
await writeFile(file, '')
|
||||
await utimes(file, 1_000_000 + index, 1_000_000 + index)
|
||||
}
|
||||
for (let dependency = 0; dependency < DEPENDENCIES; dependency++) {
|
||||
const directory = join(root, 'node_modules', `dependency-${dependency}`, 'src')
|
||||
await mkdir(directory, { recursive: true })
|
||||
await Promise.all(Array.from({ length: FILES_PER_DEPENDENCY }, (_, index) => writeFile(join(directory, `file-${index}.ts`), '')))
|
||||
}
|
||||
}
|
||||
await mountAgentLoopTestDependencies(ctx, { workingDirectory: true, tools: { mode: 'native' } })
|
||||
await ctx.plugin(LocalSubprocessRuntime)
|
||||
await ctx.plugin(search, { sampleOverCapGlobResults: true })
|
||||
const harness = await mountAgentLoopTestHarness(ctx)
|
||||
const agent = await harness.create(SessionId('native-glob-perf'), {}, { cwd: root })
|
||||
const samplesMs: number[] = []
|
||||
let outputHash: string | undefined
|
||||
for (let sample = 0; sample < SAMPLES; sample++) {
|
||||
const signal = AbortSignal.timeout(60_000)
|
||||
const start = performance.now()
|
||||
const batch = await ctx.tools.execute({
|
||||
agent,
|
||||
signal,
|
||||
calls: [{ type: 'tool-call', id: ToolCallId(`glob-${sample}`), name: 'glob', arguments: JSON.stringify({ pattern: `${TARGET}/*.ts` }) }],
|
||||
})
|
||||
const elapsed = performance.now() - start
|
||||
const result = batch.results[0]?.result
|
||||
assert(result !== undefined && !result.isError, 'Native glob must complete successfully')
|
||||
assert.equal(signal.aborted, false)
|
||||
if (temporary) assert.deepEqual(result.value, { root: '.', paths: Array.from({ length: MATCHES }, (_, index) => join(TARGET, `file-${index}.ts`)) })
|
||||
const digest = createHash('sha256').update(JSON.stringify([result.value, result.content, result.meta])).digest('hex')
|
||||
outputHash ??= digest
|
||||
assert.equal(digest, outputHash, 'Every sample must return the same value, text, and presentation metadata')
|
||||
samplesMs.push(elapsed)
|
||||
process.stdout.write(JSON.stringify({ sample: sample + 1, ms: elapsed }) + '\n')
|
||||
}
|
||||
process.stdout.write(JSON.stringify({
|
||||
node: process.version, platform: `${process.platform}-${process.arch}`, cpu: cpus()[0]?.model,
|
||||
workload: temporary ? { unrelatedFiles: DEPENDENCIES * FILES_PER_DEPENDENCY, matches: MATCHES } : 'explicit workspace',
|
||||
executor: 'native', samplesMs, medianMs: samplesMs.toSorted((a, b) => a - b)[Math.floor(SAMPLES / 2)], outputHash,
|
||||
}) + '\n')
|
||||
} finally {
|
||||
try { await ctx.fiber.dispose() } finally { if (temporary) await rm(root, { recursive: true, force: true }) }
|
||||
}
|
||||
}
|
||||
|
||||
await measure()
|
||||
Reference in new issue
Block a user