perf(fs): prune glob traversal using ripgrep directory filters
This commit is contained in:
1 parent
12f63eaf89
commit
b1c8c9bbbc
5 files changed
+39
-47
No files matched your search
@@ -42,8 +42,8 @@
|
||||
en: 5a30f023dd86c345
|
||||
zh: 558b6e80c62648ba
|
||||
/deepseek-ai-dsh-tool-fs-search/understand-the-implementation/how-a-search-runs:
|
||||
en: 47f4c091ab3c0a71
|
||||
zh: 1afaab48629b6b39
|
||||
en: bffeacf8482af3f7
|
||||
zh: 9754972cab5251fd
|
||||
/deepseek-ai-dsh-tool-fs-search/understand-the-implementation/native-mode-benchmark:
|
||||
en: 031654f59e63d07e
|
||||
zh: a3b488fb1880d2f5
|
||||
|
||||
@@ -105,9 +105,9 @@ Local workspace discovery is naturally a process-backed `rg` workflow, and putti
|
||||
|
||||
### How a search runs
|
||||
|
||||
Without an explicit `path`, `glob` narrows traversal for patterns such as `packages/core/src/*.ts` when every directory component is a plain relative name that exactly matches an existing non-symlink directory outside VCS metadata. A prefix consisting only of `-` retains workspace traversal because ripgrep interprets that argument as standard input. Other patterns retain workspace traversal; matching, modification-time ordering, and the logical result root stay unchanged.
|
||||
Without an explicit `path`, `glob` prunes unrelated subtrees for patterns such as `packages/core/src/*.ts` when every directory component is a plain relative name outside VCS metadata. Ordered ripgrep globs exclude the tree, admit only the prefix directories, then apply the original file pattern. Directory-only includes cannot admit files with the same names. The search root, path separators, symlink handling, matching, and modification-time ordering stay unchanged; other patterns retain the original traversal.
|
||||
|
||||
Each call resolves its working directory once before constructing argv, then resolves the packaged binary (`@vscode/ripgrep`, or the executable's `-rg` sidecar in a pkg single-file runtime), prepends `--no-config` so a host `RIPGREP_CONFIG_PATH` cannot inject a `--pre` preprocessor into the unconfined spawn, and passes every model-controlled value as a plain argv element — no shell layer exists, so no quoting applies. Collect-mode budgets bound complete stdout and a stderr tail; a lossy stdout read fails as `SEARCH_RAW_OUTPUT_OVERFLOW` rather than parsing a silently-partial stream. The tools never read a raw spill path.
|
||||
Each call resolves the packaged binary (`@vscode/ripgrep`, or the executable's `-rg` sidecar in a pkg single-file runtime), prepends `--no-config` so a host `RIPGREP_CONFIG_PATH` cannot inject a `--pre` preprocessor into the unconfined spawn, and passes every model-controlled value as a plain argv element — no shell layer exists, so no quoting applies. Collect-mode budgets bound complete stdout and a stderr tail; a lossy stdout read fails as `SEARCH_RAW_OUTPUT_OVERFLOW` rather than parsing a silently-partial stream. The tools never read a raw spill path.
|
||||
|
||||
### Native-mode benchmark
|
||||
|
||||
|
||||
@@ -105,9 +105,9 @@ Node 部署在受支持的 macOS、Linux 与 Windows 目标上获得 `@vscode/ri
|
||||
|
||||
### 搜索如何运行
|
||||
|
||||
未显式提供 `path` 时,如果目录部分的每一项都是普通相对名称,且精确匹配已存在的非符号链接目录、不属于 VCS 元数据目录,`glob` 会为 `packages/core/src/*.ts` 这样的模式缩小遍历范围。仅由 `-` 组成的前缀仍遍历工作区,因为 ripgrep 将该参数解释为标准输入。其他模式仍遍历工作区;匹配规则、修改时间排序和结果的逻辑根目录保持不变。
|
||||
未显式提供 `path` 时,如果目录部分的每一项都是普通相对名称且不属于 VCS 元数据目录,`glob` 会为 `packages/core/src/*.ts` 这样的模式跳过无关子树。按顺序添加的 ripgrep glob 先排除目录树,再仅放行前缀目录,最后应用原始文件模式。仅匹配目录的放行规则不会纳入同名文件。搜索根目录、路径分隔符、符号链接处理、匹配规则和修改时间排序保持不变;其他模式仍按原方式遍历。
|
||||
|
||||
每次调用在构造 argv 前解析一次工作目录,然后解析打包二进制(`@vscode/ripgrep`,或 pkg 单文件运行时中可执行程序的 `-rg` 伴随文件),前置 `--no-config`,使宿主的 `RIPGREP_CONFIG_PATH` 无法向不受约束的 spawn 注入 `--pre` 预处理器,并把每个模型控制的值作为普通 argv 元素传入——不存在 shell 层,因此不涉及 shell 引号处理。collect 模式预算限制完整 stdout 与 stderr 尾部;lossy stdout 读取以 `SEARCH_RAW_OUTPUT_OVERFLOW` 失败,而不是解析静默不完整的流。工具从不读取原始 spill 路径。
|
||||
每次调用解析打包二进制(`@vscode/ripgrep`,或 pkg 单文件运行时中可执行程序的 `-rg` 伴随文件),前置 `--no-config`,使宿主的 `RIPGREP_CONFIG_PATH` 无法向不受约束的 spawn 注入 `--pre` 预处理器,并把每个模型控制的值作为普通 argv 元素传入——不存在 shell 层,因此不涉及 shell 引号处理。collect 模式预算限制完整 stdout 与 stderr 尾部;lossy stdout 读取以 `SEARCH_RAW_OUTPUT_OVERFLOW` 失败,而不是解析静默不完整的流。工具从不读取原始 spill 路径。
|
||||
|
||||
### Native 模式 benchmark
|
||||
|
||||
|
||||
@@ -10,12 +10,11 @@
|
||||
*/
|
||||
|
||||
import type { Context } from '@deepseek-ai/cordis'
|
||||
import { readdir } from 'node:fs/promises'
|
||||
import { join, sep } from 'node:path'
|
||||
import { sep } from 'node:path'
|
||||
import { defineTool } from '@deepseek-ai/dsh-tools'
|
||||
import type { GenericCallView, SearchResultView, ToolResult } from '@deepseek-ai/dsh-tools'
|
||||
import type { SpillRef } from '@deepseek-ai/dsh-spill'
|
||||
import { resolveSearchWorkdir, runRipgrep, toWorkdirRelative, trySaveFormattedResult } from './search-core.ts'
|
||||
import { runRipgrep, toWorkdirRelative, trySaveFormattedResult } from './search-core.ts'
|
||||
import { globSearchMeta, searchViewFromMeta } from './presentation.ts'
|
||||
import { acceptedDirectCallValue } from './direct-call.ts'
|
||||
|
||||
@@ -82,14 +81,27 @@ export function parseGlobArgs(args: { pattern: string; path?: string }): GlobInp
|
||||
* search root rides behind `--` so a leading-dash path can never be parsed as
|
||||
* a flag. `--sort=modified` orders by modification time, `--no-ignore
|
||||
* --hidden` searches ignored and hidden files, and
|
||||
* {@link GLOB_VCS_EXCLUDES} keeps VCS metadata out.
|
||||
* {@link GLOB_VCS_EXCLUDES} keeps VCS metadata out. Simple directory prefixes
|
||||
* prune unrelated subtrees through directory-only globs without changing the
|
||||
* search root, path separators, or symlink handling.
|
||||
*
|
||||
* @param input - the validated arguments.
|
||||
* @returns the complete ripgrep argument vector (excluding the binary itself).
|
||||
*/
|
||||
export function buildGlobCommand(input: GlobInput): string[] {
|
||||
const directories = input.pattern.split('/').slice(0, -1)
|
||||
const pruning: string[] = []
|
||||
if (input.path === undefined && directories.length > 0
|
||||
&& directories.every(part => /^[\w .-]+$/.test(part) && part !== '.' && part !== '..' && !GLOB_VCS_EXCLUDES.includes(part))) {
|
||||
pruning.push('--glob=!**')
|
||||
// Trailing slashes admit directories only, never files with the same names.
|
||||
for (let depth = 1; depth <= directories.length; depth++) {
|
||||
pruning.push(`--glob=/${directories.slice(0, depth).join('/')}/`)
|
||||
}
|
||||
}
|
||||
const parts = [
|
||||
'--files',
|
||||
...pruning,
|
||||
`--glob=${input.pattern}`,
|
||||
'--sort=modified',
|
||||
'--no-ignore',
|
||||
@@ -107,34 +119,6 @@ export function buildGlobCommand(input: GlobInput): string[] {
|
||||
return parts
|
||||
}
|
||||
|
||||
/**
|
||||
* Exact directory entries keep prefix narrowing case-sensitive, skip links, and
|
||||
* preserve the logical search root. Inspection requires the co-located local
|
||||
* workspace used by ripgrep; pending filesystem reads are not abortable.
|
||||
*/
|
||||
async function scopedGlobCommand(input: GlobInput, workdir: string): Promise<string[]> {
|
||||
const command = buildGlobCommand(input)
|
||||
const slash = input.pattern.lastIndexOf('/')
|
||||
if (input.path !== undefined || slash < 1) return command
|
||||
const segments = input.pattern.slice(0, slash).split('/')
|
||||
if (segments.some(part => !/^[\w .-]+$/.test(part) || part === '.' || part === '..' || GLOB_VCS_EXCLUDES.includes(part))) return command
|
||||
let prefix = ''
|
||||
for (const part of segments) {
|
||||
let directory: boolean
|
||||
try {
|
||||
directory = (await readdir(join(workdir, prefix), { withFileTypes: true }))
|
||||
.some(entry => entry.name === part && entry.isDirectory())
|
||||
} catch (_error) {
|
||||
// Prefix inspection is optional; ripgrep retains ownership of search errors.
|
||||
return command
|
||||
}
|
||||
if (!directory) return command
|
||||
prefix = join(prefix, part)
|
||||
}
|
||||
// ripgrep treats a bare '-' as stdin even after '--'.
|
||||
return prefix === '-' ? command : [...command, '--', prefix]
|
||||
}
|
||||
|
||||
/**
|
||||
* The inline page of a capped `glob` result, plus how much of the complete
|
||||
* result's top level it reaches.
|
||||
@@ -364,9 +348,7 @@ export function applyGlobTool(ctx: Context, caps: GlobToolCaps): void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const input = parseGlobArgs(args)
|
||||
const workdir = await resolveSearchWorkdir(ctx, exec, 'glob')
|
||||
const argv = await scopedGlobCommand(input, workdir)
|
||||
const run = await runRipgrep(ctx, exec, 'glob', workdir, argv, caps.rawOutputMaxBytes, caps.graceMs, caps.stderrMaxBytes)
|
||||
const run = await runRipgrep(ctx, exec, 'glob', buildGlobCommand(input), caps.rawOutputMaxBytes, caps.graceMs, caps.stderrMaxBytes)
|
||||
const root = input.path === undefined ? '.' : toWorkdirRelative(input.path, run.workdir)
|
||||
if (run.noMatches) return { root, paths: [] }
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@ describe('search tools over the real subprocess service + the packaged rg', () =
|
||||
})
|
||||
|
||||
describe('glob', () => {
|
||||
it('starts a literal-prefix search inside its directory and preserves workspace-relative results', async () => {
|
||||
it('prunes unrelated directories and preserves workspace-relative results', async () => {
|
||||
await mkdir(join(dir, 'packages', 'core', 'src', 'nested'), { recursive: true })
|
||||
await writeFile(join(dir, 'packages', 'core', 'src', 'old.ts'), '')
|
||||
await writeFile(join(dir, 'packages', 'core', 'src', '.hidden.ts'), '')
|
||||
@@ -97,22 +97,31 @@ describe('search tools over the real subprocess service + the packaged rg', () =
|
||||
join('packages', 'core', 'src', 'old.ts'),
|
||||
join('packages', 'core', 'src', '.hidden.ts'),
|
||||
] })
|
||||
expect(spawn.mock.calls[0]?.[0].argv.slice(-2)).toEqual(['--', join('packages', 'core', 'src')])
|
||||
expect(spawn.mock.calls[0]?.[0].argv.slice(3, 8)).toEqual([
|
||||
'--glob=!**', '--glob=/packages/', '--glob=/packages/core/', '--glob=/packages/core/src/', '--glob=packages/core/src/*.ts',
|
||||
])
|
||||
expect(spawn.mock.calls[0]?.[0].argv).not.toContain('--')
|
||||
})
|
||||
|
||||
it.each(['*.ts', '**/*.ts', 'src/**/*.ts', '!src/*.ts', './src/*.ts', 'src/../src/*.ts', 'missing/*.ts', 'notes.md/*.ts', '.git/*.ts'])(
|
||||
it.each(['*.ts', '**/*.ts', 'src/**/*.ts', '!src/*.ts', './src/*.ts', 'src/../src/*.ts', '.git/*.ts'])(
|
||||
'keeps the original traversal for %s', async (pattern) => {
|
||||
const spawn = vi.spyOn(ctx.subprocess, 'spawn')
|
||||
const result = await call('glob', { pattern }, agent())
|
||||
expect(result.isError).toBe(false)
|
||||
expect(spawn.mock.calls[0]?.[0].argv).not.toContain('--')
|
||||
expect(spawn.mock.calls[0]?.[0].argv).not.toContain('--glob=!**')
|
||||
},
|
||||
)
|
||||
|
||||
it('keeps the original traversal when directory entry casing differs', async () => {
|
||||
it.each(['missing/*.ts', 'notes.md/*.ts', 'src/alpha.ts/*.ts'])('does not admit a missing or file prefix for %s', async (pattern) => {
|
||||
const result = await call('glob', { pattern }, agent())
|
||||
expect(result.isError).toBe(false)
|
||||
expect(text(result)).toBe('No files found')
|
||||
})
|
||||
|
||||
it('preserves case-sensitive matching while pruning directories', async () => {
|
||||
const spawn = vi.spyOn(ctx.subprocess, 'spawn')
|
||||
expect(text(await call('glob', { pattern: 'SRC/*.ts' }, agent()))).toBe('No files found')
|
||||
expect(spawn.mock.calls[0]?.[0].argv).not.toContain('--')
|
||||
expect(spawn.mock.calls[0]?.[0].argv).toContain('--glob=/SRC/')
|
||||
})
|
||||
|
||||
it('does not follow a directory link when narrowing a literal prefix', async () => {
|
||||
@@ -148,6 +157,7 @@ describe('search tools over the real subprocess service + the packaged rg', () =
|
||||
const argv = spawn.mock.calls[0]?.[0].argv
|
||||
expect(argv?.filter(part => part === '--')).toHaveLength(1)
|
||||
expect(argv?.slice(-2)).toEqual(['--', '.'])
|
||||
expect(argv).not.toContain('--glob=!**')
|
||||
})
|
||||
|
||||
it('discovers files in a directory named dash without treating it as stdin', async () => {
|
||||
|
||||
Reference in new issue
Block a user