fix(scripts): enumerate repo files with a dirent walker instead of node fs.glob

Node's internal fs.glob, from some 24.x releases, lstat-probes
<matched>/<next segment> while expanding a ** pattern over a symlinked
file and throws ENOTDIR instead of skipping (observed on node 24.13.0
scanning snapshots/acp/image-compaction's symlinked
system-prompt.expected.md, which failed verify-md-wrap inside
check:ci:static). uniqueRepoFiles now walks with dirent types, never
probing a path under a file, and mirrors fs.glob's dot:false and
follow:false semantics: the four repo-files gate pattern sets enumerate
identically. Broken or cyclic symlinks fail loudly in the caller's
realpathSync, and pattern forms the walker does not model (metacharacters,
a trailing **, empty segments) are rejected up front. repo-files.spec.ts
pins the symlink, broken-link, and rejection behavior.
This commit is contained in:
Chinesezjc committed 2026-09-09 19:20:29 +08:00
1 parent 9bae613128
commit d79294f23a
2 files changed
+346 -4

No files matched your search

+176
View File
@@ -0,0 +1,176 @@
import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join, relative } from 'node:path'
import { describe, expect, it } from 'vitest'
import { uniqueRepoFiles } from './repo-files.ts'
interface Tree {
root: string
clean: () => void
}
function makeTree(): Tree {
const parent = mkdtempSync(join(tmpdir(), 'repo-files-'))
const root = join(parent, 'root')
const outside = join(parent, 'outside') // reachable only through a symlinked dir
mkdirSync(join(root, 'a'), { recursive: true })
writeFileSync(join(root, 'a', 'snap.md'), 'real\n')
mkdirSync(join(root, 'd'), { recursive: true })
symlinkSync(join(root, 'a', 'snap.md'), join(root, 'd', 'snap.md'))
mkdirSync(outside, { recursive: true })
writeFileSync(join(outside, 'snap.md'), 'behind a symlinked dir\n')
symlinkSync(outside, join(root, 'linked-dir'))
mkdirSync(join(root, '.hidden'), { recursive: true })
writeFileSync(join(root, '.hidden', 'snap.md'), 'hidden by dot\n')
return { root, clean: () => { rmSync(parent, { recursive: true, force: true }) } }
}
describe('uniqueRepoFiles', () => {
it('enumerates ** matches without probing a symlinked file as a directory', () => {
const tree = makeTree()
try {
// Node's internal glob (from some 24.x releases) lstat-probes
// <symlink>/snap.md while expanding `**/snap.md` and throws ENOTDIR.
// The walker must return the real files, deduplicated by canonical
// target, without throwing on any node version.
const files = uniqueRepoFiles(tree.root, ['**/snap.md'])
const rootReal = realpathSync(tree.root)
const reals = files.map(file => relative(rootReal, file.real)).sort()
expect(reals).toEqual([join('a', 'snap.md')])
} finally {
tree.clean()
}
})
it('does not follow symlinked directories under ** or wildcard-match dot names', () => {
const tree = makeTree()
try {
const files = uniqueRepoFiles(tree.root, ['**/*.md'])
const paths = files.map(file => relative(tree.root, file.abs)).sort()
// The symlinked d/snap.md dedupes onto its a/snap.md target; the
// linked-dir and .hidden targets must not appear at all.
expect(paths).toEqual([join('a', 'snap.md')])
} finally {
tree.clean()
}
})
it('follows a literal segment that names a symlinked directory', () => {
const tree = makeTree()
try {
// Node glob resolves literal segments with stat, so `linked-dir/**` and
// `linked-dir/*` enter the symlinked directory's target; `**` and
// wildcard segments resolve with dirent types and do not.
const files = uniqueRepoFiles(tree.root, ['linked-dir/**/*.md'])
expect(files.map(file => relative(tree.root, file.abs))).toEqual([join('linked-dir', 'snap.md')])
expect(uniqueRepoFiles(tree.root, ['linked-dir/*.md']).map(file => relative(tree.root, file.abs)))
.toEqual([join('linked-dir', 'snap.md')])
// A wildcard first segment never enters the symlinked directory.
expect(uniqueRepoFiles(tree.root, ['*/snap.md']).map(file => relative(tree.root, file.abs)))
.toEqual([join('a', 'snap.md')])
} finally {
tree.clean()
}
})
it('follows repeated literal symlinked directories like node glob and terminates', () => {
const parent = mkdtempSync(join(tmpdir(), 'repo-files-cycle-'))
try {
const root = join(parent, 'root')
mkdirSync(join(root, 'a'), { recursive: true })
writeFileSync(join(root, 'a', 'snap.md'), 'x\n')
symlinkSync(root, join(root, 'cyc'))
// Each literal `cyc` segment resolves through stat and enters the
// symlinked directory again, exactly as node glob does for a repeated
// literal; recursion stays bounded because each literal consumes one
// pattern segment and `**` only enters real directories.
expect(uniqueRepoFiles(root, ['cyc/**/snap.md']).map(file => relative(root, file.abs)))
.toEqual([join('cyc', 'a', 'snap.md')])
expect(uniqueRepoFiles(root, ['cyc/cyc/**/snap.md']).map(file => relative(root, file.abs)))
.toEqual([join('cyc', 'cyc', 'a', 'snap.md')])
expect(uniqueRepoFiles(root, ['cyc/cyc/cyc/**/snap.md']).map(file => relative(root, file.abs)))
.toEqual([join('cyc', 'cyc', 'cyc', 'a', 'snap.md')])
} finally {
rmSync(parent, { recursive: true, force: true })
}
})
it('reports a matched file canonical target for downstream existence checks', () => {
const tree = makeTree()
try {
const files = uniqueRepoFiles(tree.root, ['a/*.md'])
expect(files).toHaveLength(1)
expect(files[0]!.real).toBe(join(realpathSync(tree.root), 'a', 'snap.md'))
} finally {
tree.clean()
}
})
it('fails loudly on a broken symlink instead of shrinking the corpus', () => {
const tree = makeTree()
try {
// A healthy tree with symlinked files must not throw.
expect(() => uniqueRepoFiles(tree.root, ['a/*.md'])).not.toThrow()
} finally {
tree.clean()
}
const parent = mkdtempSync(join(tmpdir(), 'repo-files-broken-'))
try {
const root = join(parent, 'root')
mkdirSync(root, { recursive: true })
writeFileSync(join(root, 'gone.md'), 'x\n')
symlinkSync(join(root, 'gone.md'), join(root, 'broken-link.md'))
rmSync(join(root, 'gone.md'))
// realpathSync on the matched broken link must throw, exactly as it did
// under the node-glob implementation, instead of silently shrinking the
// scanned corpus.
expect(() => uniqueRepoFiles(root, ['*.md'])).toThrow()
// A broken symlink under a literal non-final segment matches nothing,
// again as node glob silently returns no match for it.
symlinkSync(join(root, 'gone-dir'), join(root, 'broken-dir'))
expect(uniqueRepoFiles(root, ['broken-dir/*.md'])).toEqual([])
} finally {
rmSync(parent, { recursive: true, force: true })
}
})
it('rejects glob syntax the walker does not model instead of matching nothing', () => {
const tree = makeTree()
try {
// Node glob would interpret the bracket class; the walker must fail
// loudly rather than expand the pattern as a literal and quietly match
// nothing.
expect(() => uniqueRepoFiles(tree.root, ['**/*.[cm]d'])).toThrow(/does not model glob syntax/)
} finally {
tree.clean()
}
})
it('rejects a trailing ** segment instead of silently returning nothing', () => {
const tree = makeTree()
try {
expect(() => uniqueRepoFiles(tree.root, ['a/**'])).toThrow(/trailing \*\*/)
// A trailing slash yields an empty final segment that can never match.
expect(() => uniqueRepoFiles(tree.root, ['a/**/'])).toThrow(/empty segments/)
} finally {
tree.clean()
}
})
it('folds a . segment and rejects a .. segment like node glob semantics', () => {
const tree = makeTree()
try {
// Node glob normalizes a `.` segment away, so `./a/snap.md` matches
// a/snap.md instead of looking for a directory named `.`.
expect(uniqueRepoFiles(tree.root, ['./a/snap.md']).map(file => relative(tree.root, file.abs)))
.toEqual([join('a', 'snap.md')])
expect(uniqueRepoFiles(tree.root, ['a/./snap.md']).map(file => relative(tree.root, file.abs)))
.toEqual([join('a', 'snap.md')])
// A `..` segment escapes the scanned root; the walker must fail loudly
// rather than silently match nothing.
expect(() => uniqueRepoFiles(tree.root, ['a/../snap.md'])).toThrow(/does not model \.\. segments/)
} finally {
tree.clean()
}
})
})
+170 -4
View File
@@ -1,7 +1,7 @@
/** Shared repository file discovery and line-oriented reference scanning. */
import { globSync, readFileSync, realpathSync } from 'node:fs'
import { relative, resolve, sep } from 'node:path'
import { readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
import { join, relative, resolve, sep } from 'node:path'
/** One authored path plus its canonical target for symlink deduplication. */
export interface RepoFile {
@@ -26,6 +26,173 @@ export function isArchivedAgentNotePath(path: string): boolean {
return path.replaceAll('\\', '/').startsWith('.agents/notes/archived/')
}
/**
* Whether a pattern segment matches a directory or file name. Supports `*` and
* `?` inside a segment and mirrors node's glob `dot: false`: a segment whose
* first character is a wildcard does not match dot names. `**` is handled as a
* whole segment by the walker, never here.
*/
function segmentMatches(pattern: string, name: string): boolean {
if (name.startsWith('.') && (pattern.startsWith('*') || pattern.startsWith('?'))) return false
let expression = ''
for (const character of pattern) {
if (character === '*') expression += '.*'
else if (character === '?') expression += '.'
else expression += character.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
}
return new RegExp(`^${expression}$`).test(name)
}
/**
* Reject glob syntax the walker does not model. Node's `fs.glob` understands
* character classes, brace alternation, and extglobs; expanding those
* silently as literals would match nothing and quietly shrink a gate's
* corpus, so a pattern segment using them fails loudly instead. Pure literal
* segments must therefore contain none of the rejected metacharacters
* either, even where node glob would read them literally.
*/
const UNSUPPORTED_GLOB = /[[\]{}()!+@\\]/
function assertSupportedSegment(segment: string): void {
if (UNSUPPORTED_GLOB.test(segment)) {
throw new TypeError(`repo-files walker does not model glob syntax in segment: ${segment}`)
}
}
/**
* Walk `root` matching one repository-relative glob without node's `fs.glob`.
* The repository's own walker exists because node's internal glob, from some
* 24.x releases, lstat-probes `<matched>/<next segment>` for symlinked files
* while expanding `**` and throws ENOTDIR instead of skipping (observed on
* node 24.13.0 scanning `snapshots/acp/image-compaction`'s symlinked
* `system-prompt.expected.md`). The walker decides directoryhood from dirent
* types and stat results, never by probing a path under a file, so the same
* tree enumerates identically on every node version.
*
* Segment semantics: literal segments resolve through `stat`, so a literal
* naming a symlinked directory enters that directory exactly as node glob
* does — including a literal repeated across segments, which node glob
* resolves each time; wildcard segments and `**` resolve through dirent
* types, so they never enter symlinked directories and never match dot
* names. Node documents `follow: false` only for `**` expansion, so the
* wildcard-side behavior is this walker's own contract, pinned by
* repo-files.spec.ts, rather than a cross-version node guarantee; the four
* consuming gates' current patterns contain no wildcard segment over a
* symlinked directory, so the corpus is unchanged. `**` spans zero or more
* directories. The final segment matches files and symlinks; a broken or
* cyclic symlink then fails loudly in the caller's realpathSync rather than
* shrinking the scanned corpus, while a broken symlink under a literal
* non-final segment matches nothing, again as node glob does. Traversal
* terminates without a visited set: each literal segment consumes one pattern
* segment per recursion, and `**` recurses only into real directories, which
* form a finite tree because symlinked directories are never expanded by it.
* Pattern segments support `*`, `?`, and literals that contain none of
* `[]{}()!+@\`; other node-glob syntax, including a trailing `**`, `..`, and
* empty segments, is rejected loudly up front, while a `.` segment is folded
* away exactly as node glob normalizes it. Returns repository-relative slash
* paths, sorted.
*/
function expandGlob(root: string, pattern: string): string[] {
const segments: string[] = []
for (const segment of pattern.split('/')) {
if (segment === '.') {
// Node glob normalizes a `.` segment away; dropping it here keeps
// `./README.md` and `a/./b` matching exactly as node glob does instead
// of silently matching nothing.
continue
}
if (segment === '..') {
// A `..` segment escapes the scanned root and interacts with `**` in
// ways node glob special-cases; no gate pattern uses one, so the walker
// rejects the form loudly instead of silently matching nothing.
throw new TypeError(`repo-files walker does not model .. segments in pattern: ${pattern}`)
}
if (segment === '') {
// A leading, doubled, or trailing slash yields an empty segment that
// can never match an entry; node glob would tolerate the form, so the
// walker must reject it loudly instead of silently returning nothing.
throw new TypeError(`repo-files walker does not model empty segments in pattern: ${pattern}`)
}
segments.push(segment)
}
if (segments.length === 0) {
throw new TypeError(`repo-files walker does not model a pattern with no segments: ${pattern}`)
}
for (const segment of segments) {
if (segment !== '**') assertSupportedSegment(segment)
}
// A trailing `**` would match files, directories, and symlinks below the
// prefix; the walker models `**` only as a directory-spanning segment, so
// it must reject the form loudly instead of silently returning nothing.
if (segments[segments.length - 1] === '**') {
throw new TypeError(`repo-files walker does not model a trailing ** segment in pattern: ${pattern}`)
}
const out: string[] = []
const visit = (dirAbs: string, dirRel: string, index: number): void => {
if (index >= segments.length) return
if (segments[index] !== '**') {
visitSegment(dirAbs, dirRel, index)
return
}
// `**` consumes zero directories here and one directory per recursion.
visitSegment(dirAbs, dirRel, index + 1)
for (const entry of readdirSync(dirAbs, { withFileTypes: true })) {
if (entry.name.startsWith('.') || !entry.isDirectory()) continue
visit(join(dirAbs, entry.name), dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`, index)
}
}
const visitSegment = (dirAbs: string, dirRel: string, index: number): void => {
if (index >= segments.length) return
const segment = segments[index]
if (segment === undefined) return
if (segment === '**') {
visit(dirAbs, dirRel, index)
return
}
const last = index === segments.length - 1
for (const entry of readdirSync(dirAbs, { withFileTypes: true })) {
if (!segmentMatches(segment, entry.name)) continue
const childAbs = join(dirAbs, entry.name)
const childRel = dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`
if (last) {
// Files and symlinks match; a broken or cyclic symlink then fails
// loudly in the caller's realpathSync exactly as node glob did,
// rather than silently shrinking the scanned corpus.
if (entry.isFile() || entry.isSymbolicLink()) out.push(childRel)
continue
}
if (entry.isDirectory()) {
visitSegment(childAbs, childRel, index + 1)
continue
}
// A literal non-final segment resolves through stat like node glob's,
// so it enters a symlinked directory; wildcard segments never reach
// this branch because they are matched from dirent types above.
if (!entry.isSymbolicLink() || hasWildcard(segment)) continue
let target: ReturnType<typeof statSync>
try {
target = statSync(childAbs)
} catch {
// A broken symlink matches nothing under a literal segment, as node
// glob silently returns no match for it.
continue
}
if (!target.isDirectory()) continue
visitSegment(childAbs, childRel, index + 1)
}
}
visit(root, '.', 0)
return out.sort()
}
/** Whether a pattern segment contains `*` or `?` and is therefore wildcard. */
function hasWildcard(segment: string): boolean {
return segment.includes('*') || segment.includes('?')
}
/**
* Expand repository-relative globs and deduplicate symlinked files.
* @param root - absolute repository root.
@@ -41,8 +208,7 @@ export function uniqueRepoFiles(
const seen = new Set<string>()
const files: RepoFile[] = []
for (const pattern of patterns) {
for (const match of globSync(pattern, { cwd: root })) {
const repoPath = match.split(sep).join('/')
for (const repoPath of expandGlob(root, pattern)) {
if (isExcluded(repoPath)) continue
const abs = resolve(root, repoPath)
const real = realpathSync(abs)