perf(fs): prune glob traversal using ripgrep directory filters

This commit is contained in:
_Kerman authored and imccyu committed 2026-10-26 12:04:34 +08:00
1 parent 12f63eaf89
commit b1c8c9bbbc
5 files changed
+39 -47

No files matched your search

+2 -2
View File
@@ -42,8 +42,8 @@
en: 5a30f023dd86c345
zh: 558b6e80c62648ba
/deepseek-ai-dsh-tool-fs-search/understand-the-implementation/how-a-search-runs:
en: 47f4c091ab3c0a71
zh: 1afaab48629b6b39
en: bffeacf8482af3f7
zh: 9754972cab5251fd
/deepseek-ai-dsh-tool-fs-search/understand-the-implementation/native-mode-benchmark:
en: 031654f59e63d07e
zh: a3b488fb1880d2f5
+2 -2
View File
@@ -105,9 +105,9 @@ Local workspace discovery is naturally a process-backed `rg` workflow, and putti
### How a search runs
Without an explicit `path`, `glob` narrows traversal for patterns such as `packages/core/src/*.ts` when every directory component is a plain relative name that exactly matches an existing non-symlink directory outside VCS metadata. A prefix consisting only of `-` retains workspace traversal because ripgrep interprets that argument as standard input. Other patterns retain workspace traversal; matching, modification-time ordering, and the logical result root stay unchanged.
Without an explicit `path`, `glob` prunes unrelated subtrees for patterns such as `packages/core/src/*.ts` when every directory component is a plain relative name outside VCS metadata. Ordered ripgrep globs exclude the tree, admit only the prefix directories, then apply the original file pattern. Directory-only includes cannot admit files with the same names. The search root, path separators, symlink handling, matching, and modification-time ordering stay unchanged; other patterns retain the original traversal.
Each call resolves its working directory once before constructing argv, then resolves the packaged binary (`@vscode/ripgrep`, or the executable's `-rg` sidecar in a pkg single-file runtime), prepends `--no-config` so a host `RIPGREP_CONFIG_PATH` cannot inject a `--pre` preprocessor into the unconfined spawn, and passes every model-controlled value as a plain argv element — no shell layer exists, so no quoting applies. Collect-mode budgets bound complete stdout and a stderr tail; a lossy stdout read fails as `SEARCH_RAW_OUTPUT_OVERFLOW` rather than parsing a silently-partial stream. The tools never read a raw spill path.
Each call resolves the packaged binary (`@vscode/ripgrep`, or the executable's `-rg` sidecar in a pkg single-file runtime), prepends `--no-config` so a host `RIPGREP_CONFIG_PATH` cannot inject a `--pre` preprocessor into the unconfined spawn, and passes every model-controlled value as a plain argv element — no shell layer exists, so no quoting applies. Collect-mode budgets bound complete stdout and a stderr tail; a lossy stdout read fails as `SEARCH_RAW_OUTPUT_OVERFLOW` rather than parsing a silently-partial stream. The tools never read a raw spill path.
### Native-mode benchmark
+2 -2
View File
@@ -105,9 +105,9 @@ Node 部署在受支持的 macOS、Linux 与 Windows 目标上获得 `@vscode/ri
### 搜索如何运行
未显式提供 `path` 时,如果目录部分的每一项都是普通相对名称,且精确匹配已存在的非符号链接目录、不属于 VCS 元数据目录,`glob` 会为 `packages/core/src/*.ts` 这样的模式缩小遍历范围。仅由 `-` 组成的前缀仍遍历工作区,因为 ripgrep 将该参数解释为标准输入。其他模式仍遍历工作区;匹配规则、修改时间排序和结果的逻辑根目录保持不变。
未显式提供 `path` 时,如果目录部分的每一项都是普通相对名称且不属于 VCS 元数据目录,`glob` 会为 `packages/core/src/*.ts` 这样的模式跳过无关子树。按顺序添加的 ripgrep glob 先排除目录树,再仅放行前缀目录,最后应用原始文件模式。仅匹配目录的放行规则不会纳入同名文件。搜索根目录、路径分隔符、符号链接处理、匹配规则和修改时间排序保持不变;其他模式仍按原方式遍历。
每次调用在构造 argv 前解析一次工作目录,然后解析打包二进制(`@vscode/ripgrep`,或 pkg 单文件运行时中可执行程序的 `-rg` 伴随文件),前置 `--no-config`,使宿主的 `RIPGREP_CONFIG_PATH` 无法向不受约束的 spawn 注入 `--pre` 预处理器,并把每个模型控制的值作为普通 argv 元素传入——不存在 shell 层,因此不涉及 shell 引号处理。collect 模式预算限制完整 stdout 与 stderr 尾部;lossy stdout 读取以 `SEARCH_RAW_OUTPUT_OVERFLOW` 失败,而不是解析静默不完整的流。工具从不读取原始 spill 路径。
每次调用解析打包二进制(`@vscode/ripgrep`,或 pkg 单文件运行时中可执行程序的 `-rg` 伴随文件),前置 `--no-config`,使宿主的 `RIPGREP_CONFIG_PATH` 无法向不受约束的 spawn 注入 `--pre` 预处理器,并把每个模型控制的值作为普通 argv 元素传入——不存在 shell 层,因此不涉及 shell 引号处理。collect 模式预算限制完整 stdout 与 stderr 尾部;lossy stdout 读取以 `SEARCH_RAW_OUTPUT_OVERFLOW` 失败,而不是解析静默不完整的流。工具从不读取原始 spill 路径。
### Native 模式 benchmark
+17 -35
View File
@@ -10,12 +10,11 @@
*/
import type { Context } from '@deepseek-ai/cordis'
import { readdir } from 'node:fs/promises'
import { join, sep } from 'node:path'
import { sep } from 'node:path'
import { defineTool } from '@deepseek-ai/dsh-tools'
import type { GenericCallView, SearchResultView, ToolResult } from '@deepseek-ai/dsh-tools'
import type { SpillRef } from '@deepseek-ai/dsh-spill'
import { resolveSearchWorkdir, runRipgrep, toWorkdirRelative, trySaveFormattedResult } from './search-core.ts'
import { runRipgrep, toWorkdirRelative, trySaveFormattedResult } from './search-core.ts'
import { globSearchMeta, searchViewFromMeta } from './presentation.ts'
import { acceptedDirectCallValue } from './direct-call.ts'
@@ -82,14 +81,27 @@ export function parseGlobArgs(args: { pattern: string; path?: string }): GlobInp
* search root rides behind `--` so a leading-dash path can never be parsed as
* a flag. `--sort=modified` orders by modification time, `--no-ignore
* --hidden` searches ignored and hidden files, and
* {@link GLOB_VCS_EXCLUDES} keeps VCS metadata out.
* {@link GLOB_VCS_EXCLUDES} keeps VCS metadata out. Simple directory prefixes
* prune unrelated subtrees through directory-only globs without changing the
* search root, path separators, or symlink handling.
*
* @param input - the validated arguments.
* @returns the complete ripgrep argument vector (excluding the binary itself).
*/
export function buildGlobCommand(input: GlobInput): string[] {
const directories = input.pattern.split('/').slice(0, -1)
const pruning: string[] = []
if (input.path === undefined && directories.length > 0
&& directories.every(part => /^[\w .-]+$/.test(part) && part !== '.' && part !== '..' && !GLOB_VCS_EXCLUDES.includes(part))) {
pruning.push('--glob=!**')
// Trailing slashes admit directories only, never files with the same names.
for (let depth = 1; depth <= directories.length; depth++) {
pruning.push(`--glob=/${directories.slice(0, depth).join('/')}/`)
}
}
const parts = [
'--files',
...pruning,
`--glob=${input.pattern}`,
'--sort=modified',
'--no-ignore',
@@ -107,34 +119,6 @@ export function buildGlobCommand(input: GlobInput): string[] {
return parts
}
/**
* Exact directory entries keep prefix narrowing case-sensitive, skip links, and
* preserve the logical search root. Inspection requires the co-located local
* workspace used by ripgrep; pending filesystem reads are not abortable.
*/
async function scopedGlobCommand(input: GlobInput, workdir: string): Promise<string[]> {
const command = buildGlobCommand(input)
const slash = input.pattern.lastIndexOf('/')
if (input.path !== undefined || slash < 1) return command
const segments = input.pattern.slice(0, slash).split('/')
if (segments.some(part => !/^[\w .-]+$/.test(part) || part === '.' || part === '..' || GLOB_VCS_EXCLUDES.includes(part))) return command
let prefix = ''
for (const part of segments) {
let directory: boolean
try {
directory = (await readdir(join(workdir, prefix), { withFileTypes: true }))
.some(entry => entry.name === part && entry.isDirectory())
} catch (_error) {
// Prefix inspection is optional; ripgrep retains ownership of search errors.
return command
}
if (!directory) return command
prefix = join(prefix, part)
}
// ripgrep treats a bare '-' as stdin even after '--'.
return prefix === '-' ? command : [...command, '--', prefix]
}
/**
* The inline page of a capped `glob` result, plus how much of the complete
* result's top level it reaches.
@@ -364,9 +348,7 @@ export function applyGlobTool(ctx: Context, caps: GlobToolCaps): void {
},
async execute(args, exec) {
const input = parseGlobArgs(args)
const workdir = await resolveSearchWorkdir(ctx, exec, 'glob')
const argv = await scopedGlobCommand(input, workdir)
const run = await runRipgrep(ctx, exec, 'glob', workdir, argv, caps.rawOutputMaxBytes, caps.graceMs, caps.stderrMaxBytes)
const run = await runRipgrep(ctx, exec, 'glob', buildGlobCommand(input), caps.rawOutputMaxBytes, caps.graceMs, caps.stderrMaxBytes)
const root = input.path === undefined ? '.' : toWorkdirRelative(input.path, run.workdir)
if (run.noMatches) return { root, paths: [] }
@@ -80,7 +80,7 @@ describe('search tools over the real subprocess service + the packaged rg', () =
})
describe('glob', () => {
it('starts a literal-prefix search inside its directory and preserves workspace-relative results', async () => {
it('prunes unrelated directories and preserves workspace-relative results', async () => {
await mkdir(join(dir, 'packages', 'core', 'src', 'nested'), { recursive: true })
await writeFile(join(dir, 'packages', 'core', 'src', 'old.ts'), '')
await writeFile(join(dir, 'packages', 'core', 'src', '.hidden.ts'), '')
@@ -97,22 +97,31 @@ describe('search tools over the real subprocess service + the packaged rg', () =
join('packages', 'core', 'src', 'old.ts'),
join('packages', 'core', 'src', '.hidden.ts'),
] })
expect(spawn.mock.calls[0]?.[0].argv.slice(-2)).toEqual(['--', join('packages', 'core', 'src')])
expect(spawn.mock.calls[0]?.[0].argv.slice(3, 8)).toEqual([
'--glob=!**', '--glob=/packages/', '--glob=/packages/core/', '--glob=/packages/core/src/', '--glob=packages/core/src/*.ts',
])
expect(spawn.mock.calls[0]?.[0].argv).not.toContain('--')
})
it.each(['*.ts', '**/*.ts', 'src/**/*.ts', '!src/*.ts', './src/*.ts', 'src/../src/*.ts', 'missing/*.ts', 'notes.md/*.ts', '.git/*.ts'])(
it.each(['*.ts', '**/*.ts', 'src/**/*.ts', '!src/*.ts', './src/*.ts', 'src/../src/*.ts', '.git/*.ts'])(
'keeps the original traversal for %s', async (pattern) => {
const spawn = vi.spyOn(ctx.subprocess, 'spawn')
const result = await call('glob', { pattern }, agent())
expect(result.isError).toBe(false)
expect(spawn.mock.calls[0]?.[0].argv).not.toContain('--')
expect(spawn.mock.calls[0]?.[0].argv).not.toContain('--glob=!**')
},
)
it('keeps the original traversal when directory entry casing differs', async () => {
it.each(['missing/*.ts', 'notes.md/*.ts', 'src/alpha.ts/*.ts'])('does not admit a missing or file prefix for %s', async (pattern) => {
const result = await call('glob', { pattern }, agent())
expect(result.isError).toBe(false)
expect(text(result)).toBe('No files found')
})
it('preserves case-sensitive matching while pruning directories', async () => {
const spawn = vi.spyOn(ctx.subprocess, 'spawn')
expect(text(await call('glob', { pattern: 'SRC/*.ts' }, agent()))).toBe('No files found')
expect(spawn.mock.calls[0]?.[0].argv).not.toContain('--')
expect(spawn.mock.calls[0]?.[0].argv).toContain('--glob=/SRC/')
})
it('does not follow a directory link when narrowing a literal prefix', async () => {
@@ -148,6 +157,7 @@ describe('search tools over the real subprocess service + the packaged rg', () =
const argv = spawn.mock.calls[0]?.[0].argv
expect(argv?.filter(part => part === '--')).toHaveLength(1)
expect(argv?.slice(-2)).toEqual(['--', '.'])
expect(argv).not.toContain('--glob=!**')
})
it('discovers files in a directory named dash without treating it as stdin', async () => {