fix(scripts): enumerate repo files with a dirent walker instead of node fs.glob
Node's internal fs.glob, from some 24.x releases, lstat-probes <matched>/<next segment> while expanding a ** pattern over a symlinked file and throws ENOTDIR instead of skipping (observed on node 24.13.0 scanning snapshots/acp/image-compaction's symlinked system-prompt.expected.md, which failed verify-md-wrap inside check:ci:static). uniqueRepoFiles now walks with dirent types, never probing a path under a file, and mirrors fs.glob's dot:false and follow:false semantics: the four repo-files gate pattern sets enumerate identically. Broken or cyclic symlinks fail loudly in the caller's realpathSync, and pattern forms the walker does not model (metacharacters, a trailing **, empty segments) are rejected up front. repo-files.spec.ts pins the symlink, broken-link, and rejection behavior.
This commit is contained in:
1 parent
9bae613128
commit
d79294f23a
2 files changed
+346
-4
No files matched your search
@@ -0,0 +1,176 @@
|
||||
import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, relative } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { uniqueRepoFiles } from './repo-files.ts'
|
||||
|
||||
interface Tree {
|
||||
root: string
|
||||
clean: () => void
|
||||
}
|
||||
|
||||
function makeTree(): Tree {
|
||||
const parent = mkdtempSync(join(tmpdir(), 'repo-files-'))
|
||||
const root = join(parent, 'root')
|
||||
const outside = join(parent, 'outside') // reachable only through a symlinked dir
|
||||
mkdirSync(join(root, 'a'), { recursive: true })
|
||||
writeFileSync(join(root, 'a', 'snap.md'), 'real\n')
|
||||
mkdirSync(join(root, 'd'), { recursive: true })
|
||||
symlinkSync(join(root, 'a', 'snap.md'), join(root, 'd', 'snap.md'))
|
||||
mkdirSync(outside, { recursive: true })
|
||||
writeFileSync(join(outside, 'snap.md'), 'behind a symlinked dir\n')
|
||||
symlinkSync(outside, join(root, 'linked-dir'))
|
||||
mkdirSync(join(root, '.hidden'), { recursive: true })
|
||||
writeFileSync(join(root, '.hidden', 'snap.md'), 'hidden by dot\n')
|
||||
return { root, clean: () => { rmSync(parent, { recursive: true, force: true }) } }
|
||||
}
|
||||
|
||||
describe('uniqueRepoFiles', () => {
|
||||
it('enumerates ** matches without probing a symlinked file as a directory', () => {
|
||||
const tree = makeTree()
|
||||
try {
|
||||
// Node's internal glob (from some 24.x releases) lstat-probes
|
||||
// <symlink>/snap.md while expanding `**/snap.md` and throws ENOTDIR.
|
||||
// The walker must return the real files, deduplicated by canonical
|
||||
// target, without throwing on any node version.
|
||||
const files = uniqueRepoFiles(tree.root, ['**/snap.md'])
|
||||
const rootReal = realpathSync(tree.root)
|
||||
const reals = files.map(file => relative(rootReal, file.real)).sort()
|
||||
expect(reals).toEqual([join('a', 'snap.md')])
|
||||
} finally {
|
||||
tree.clean()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not follow symlinked directories under ** or wildcard-match dot names', () => {
|
||||
const tree = makeTree()
|
||||
try {
|
||||
const files = uniqueRepoFiles(tree.root, ['**/*.md'])
|
||||
const paths = files.map(file => relative(tree.root, file.abs)).sort()
|
||||
// The symlinked d/snap.md dedupes onto its a/snap.md target; the
|
||||
// linked-dir and .hidden targets must not appear at all.
|
||||
expect(paths).toEqual([join('a', 'snap.md')])
|
||||
} finally {
|
||||
tree.clean()
|
||||
}
|
||||
})
|
||||
|
||||
it('follows a literal segment that names a symlinked directory', () => {
|
||||
const tree = makeTree()
|
||||
try {
|
||||
// Node glob resolves literal segments with stat, so `linked-dir/**` and
|
||||
// `linked-dir/*` enter the symlinked directory's target; `**` and
|
||||
// wildcard segments resolve with dirent types and do not.
|
||||
const files = uniqueRepoFiles(tree.root, ['linked-dir/**/*.md'])
|
||||
expect(files.map(file => relative(tree.root, file.abs))).toEqual([join('linked-dir', 'snap.md')])
|
||||
expect(uniqueRepoFiles(tree.root, ['linked-dir/*.md']).map(file => relative(tree.root, file.abs)))
|
||||
.toEqual([join('linked-dir', 'snap.md')])
|
||||
// A wildcard first segment never enters the symlinked directory.
|
||||
expect(uniqueRepoFiles(tree.root, ['*/snap.md']).map(file => relative(tree.root, file.abs)))
|
||||
.toEqual([join('a', 'snap.md')])
|
||||
} finally {
|
||||
tree.clean()
|
||||
}
|
||||
})
|
||||
|
||||
it('follows repeated literal symlinked directories like node glob and terminates', () => {
|
||||
const parent = mkdtempSync(join(tmpdir(), 'repo-files-cycle-'))
|
||||
try {
|
||||
const root = join(parent, 'root')
|
||||
mkdirSync(join(root, 'a'), { recursive: true })
|
||||
writeFileSync(join(root, 'a', 'snap.md'), 'x\n')
|
||||
symlinkSync(root, join(root, 'cyc'))
|
||||
// Each literal `cyc` segment resolves through stat and enters the
|
||||
// symlinked directory again, exactly as node glob does for a repeated
|
||||
// literal; recursion stays bounded because each literal consumes one
|
||||
// pattern segment and `**` only enters real directories.
|
||||
expect(uniqueRepoFiles(root, ['cyc/**/snap.md']).map(file => relative(root, file.abs)))
|
||||
.toEqual([join('cyc', 'a', 'snap.md')])
|
||||
expect(uniqueRepoFiles(root, ['cyc/cyc/**/snap.md']).map(file => relative(root, file.abs)))
|
||||
.toEqual([join('cyc', 'cyc', 'a', 'snap.md')])
|
||||
expect(uniqueRepoFiles(root, ['cyc/cyc/cyc/**/snap.md']).map(file => relative(root, file.abs)))
|
||||
.toEqual([join('cyc', 'cyc', 'cyc', 'a', 'snap.md')])
|
||||
} finally {
|
||||
rmSync(parent, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('reports a matched file canonical target for downstream existence checks', () => {
|
||||
const tree = makeTree()
|
||||
try {
|
||||
const files = uniqueRepoFiles(tree.root, ['a/*.md'])
|
||||
expect(files).toHaveLength(1)
|
||||
expect(files[0]!.real).toBe(join(realpathSync(tree.root), 'a', 'snap.md'))
|
||||
} finally {
|
||||
tree.clean()
|
||||
}
|
||||
})
|
||||
|
||||
it('fails loudly on a broken symlink instead of shrinking the corpus', () => {
|
||||
const tree = makeTree()
|
||||
try {
|
||||
// A healthy tree with symlinked files must not throw.
|
||||
expect(() => uniqueRepoFiles(tree.root, ['a/*.md'])).not.toThrow()
|
||||
} finally {
|
||||
tree.clean()
|
||||
}
|
||||
const parent = mkdtempSync(join(tmpdir(), 'repo-files-broken-'))
|
||||
try {
|
||||
const root = join(parent, 'root')
|
||||
mkdirSync(root, { recursive: true })
|
||||
writeFileSync(join(root, 'gone.md'), 'x\n')
|
||||
symlinkSync(join(root, 'gone.md'), join(root, 'broken-link.md'))
|
||||
rmSync(join(root, 'gone.md'))
|
||||
// realpathSync on the matched broken link must throw, exactly as it did
|
||||
// under the node-glob implementation, instead of silently shrinking the
|
||||
// scanned corpus.
|
||||
expect(() => uniqueRepoFiles(root, ['*.md'])).toThrow()
|
||||
// A broken symlink under a literal non-final segment matches nothing,
|
||||
// again as node glob silently returns no match for it.
|
||||
symlinkSync(join(root, 'gone-dir'), join(root, 'broken-dir'))
|
||||
expect(uniqueRepoFiles(root, ['broken-dir/*.md'])).toEqual([])
|
||||
} finally {
|
||||
rmSync(parent, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects glob syntax the walker does not model instead of matching nothing', () => {
|
||||
const tree = makeTree()
|
||||
try {
|
||||
// Node glob would interpret the bracket class; the walker must fail
|
||||
// loudly rather than expand the pattern as a literal and quietly match
|
||||
// nothing.
|
||||
expect(() => uniqueRepoFiles(tree.root, ['**/*.[cm]d'])).toThrow(/does not model glob syntax/)
|
||||
} finally {
|
||||
tree.clean()
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects a trailing ** segment instead of silently returning nothing', () => {
|
||||
const tree = makeTree()
|
||||
try {
|
||||
expect(() => uniqueRepoFiles(tree.root, ['a/**'])).toThrow(/trailing \*\*/)
|
||||
// A trailing slash yields an empty final segment that can never match.
|
||||
expect(() => uniqueRepoFiles(tree.root, ['a/**/'])).toThrow(/empty segments/)
|
||||
} finally {
|
||||
tree.clean()
|
||||
}
|
||||
})
|
||||
|
||||
it('folds a . segment and rejects a .. segment like node glob semantics', () => {
|
||||
const tree = makeTree()
|
||||
try {
|
||||
// Node glob normalizes a `.` segment away, so `./a/snap.md` matches
|
||||
// a/snap.md instead of looking for a directory named `.`.
|
||||
expect(uniqueRepoFiles(tree.root, ['./a/snap.md']).map(file => relative(tree.root, file.abs)))
|
||||
.toEqual([join('a', 'snap.md')])
|
||||
expect(uniqueRepoFiles(tree.root, ['a/./snap.md']).map(file => relative(tree.root, file.abs)))
|
||||
.toEqual([join('a', 'snap.md')])
|
||||
// A `..` segment escapes the scanned root; the walker must fail loudly
|
||||
// rather than silently match nothing.
|
||||
expect(() => uniqueRepoFiles(tree.root, ['a/../snap.md'])).toThrow(/does not model \.\. segments/)
|
||||
} finally {
|
||||
tree.clean()
|
||||
}
|
||||
})
|
||||
})
|
||||
+170
-4
@@ -1,7 +1,7 @@
|
||||
/** Shared repository file discovery and line-oriented reference scanning. */
|
||||
|
||||
import { globSync, readFileSync, realpathSync } from 'node:fs'
|
||||
import { relative, resolve, sep } from 'node:path'
|
||||
import { readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
|
||||
import { join, relative, resolve, sep } from 'node:path'
|
||||
|
||||
/** One authored path plus its canonical target for symlink deduplication. */
|
||||
export interface RepoFile {
|
||||
@@ -26,6 +26,173 @@ export function isArchivedAgentNotePath(path: string): boolean {
|
||||
return path.replaceAll('\\', '/').startsWith('.agents/notes/archived/')
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a pattern segment matches a directory or file name. Supports `*` and
|
||||
* `?` inside a segment and mirrors node's glob `dot: false`: a segment whose
|
||||
* first character is a wildcard does not match dot names. `**` is handled as a
|
||||
* whole segment by the walker, never here.
|
||||
*/
|
||||
function segmentMatches(pattern: string, name: string): boolean {
|
||||
if (name.startsWith('.') && (pattern.startsWith('*') || pattern.startsWith('?'))) return false
|
||||
let expression = ''
|
||||
for (const character of pattern) {
|
||||
if (character === '*') expression += '.*'
|
||||
else if (character === '?') expression += '.'
|
||||
else expression += character.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
}
|
||||
return new RegExp(`^${expression}$`).test(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject glob syntax the walker does not model. Node's `fs.glob` understands
|
||||
* character classes, brace alternation, and extglobs; expanding those
|
||||
* silently as literals would match nothing and quietly shrink a gate's
|
||||
* corpus, so a pattern segment using them fails loudly instead. Pure literal
|
||||
* segments must therefore contain none of the rejected metacharacters
|
||||
* either, even where node glob would read them literally.
|
||||
*/
|
||||
const UNSUPPORTED_GLOB = /[[\]{}()!+@\\]/
|
||||
|
||||
function assertSupportedSegment(segment: string): void {
|
||||
if (UNSUPPORTED_GLOB.test(segment)) {
|
||||
throw new TypeError(`repo-files walker does not model glob syntax in segment: ${segment}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk `root` matching one repository-relative glob without node's `fs.glob`.
|
||||
* The repository's own walker exists because node's internal glob, from some
|
||||
* 24.x releases, lstat-probes `<matched>/<next segment>` for symlinked files
|
||||
* while expanding `**` and throws ENOTDIR instead of skipping (observed on
|
||||
* node 24.13.0 scanning `snapshots/acp/image-compaction`'s symlinked
|
||||
* `system-prompt.expected.md`). The walker decides directoryhood from dirent
|
||||
* types and stat results, never by probing a path under a file, so the same
|
||||
* tree enumerates identically on every node version.
|
||||
*
|
||||
* Segment semantics: literal segments resolve through `stat`, so a literal
|
||||
* naming a symlinked directory enters that directory exactly as node glob
|
||||
* does — including a literal repeated across segments, which node glob
|
||||
* resolves each time; wildcard segments and `**` resolve through dirent
|
||||
* types, so they never enter symlinked directories and never match dot
|
||||
* names. Node documents `follow: false` only for `**` expansion, so the
|
||||
* wildcard-side behavior is this walker's own contract, pinned by
|
||||
* repo-files.spec.ts, rather than a cross-version node guarantee; the four
|
||||
* consuming gates' current patterns contain no wildcard segment over a
|
||||
* symlinked directory, so the corpus is unchanged. `**` spans zero or more
|
||||
* directories. The final segment matches files and symlinks; a broken or
|
||||
* cyclic symlink then fails loudly in the caller's realpathSync rather than
|
||||
* shrinking the scanned corpus, while a broken symlink under a literal
|
||||
* non-final segment matches nothing, again as node glob does. Traversal
|
||||
* terminates without a visited set: each literal segment consumes one pattern
|
||||
* segment per recursion, and `**` recurses only into real directories, which
|
||||
* form a finite tree because symlinked directories are never expanded by it.
|
||||
* Pattern segments support `*`, `?`, and literals that contain none of
|
||||
* `[]{}()!+@\`; other node-glob syntax, including a trailing `**`, `..`, and
|
||||
* empty segments, is rejected loudly up front, while a `.` segment is folded
|
||||
* away exactly as node glob normalizes it. Returns repository-relative slash
|
||||
* paths, sorted.
|
||||
*/
|
||||
function expandGlob(root: string, pattern: string): string[] {
|
||||
const segments: string[] = []
|
||||
for (const segment of pattern.split('/')) {
|
||||
if (segment === '.') {
|
||||
// Node glob normalizes a `.` segment away; dropping it here keeps
|
||||
// `./README.md` and `a/./b` matching exactly as node glob does instead
|
||||
// of silently matching nothing.
|
||||
continue
|
||||
}
|
||||
if (segment === '..') {
|
||||
// A `..` segment escapes the scanned root and interacts with `**` in
|
||||
// ways node glob special-cases; no gate pattern uses one, so the walker
|
||||
// rejects the form loudly instead of silently matching nothing.
|
||||
throw new TypeError(`repo-files walker does not model .. segments in pattern: ${pattern}`)
|
||||
}
|
||||
if (segment === '') {
|
||||
// A leading, doubled, or trailing slash yields an empty segment that
|
||||
// can never match an entry; node glob would tolerate the form, so the
|
||||
// walker must reject it loudly instead of silently returning nothing.
|
||||
throw new TypeError(`repo-files walker does not model empty segments in pattern: ${pattern}`)
|
||||
}
|
||||
segments.push(segment)
|
||||
}
|
||||
if (segments.length === 0) {
|
||||
throw new TypeError(`repo-files walker does not model a pattern with no segments: ${pattern}`)
|
||||
}
|
||||
for (const segment of segments) {
|
||||
if (segment !== '**') assertSupportedSegment(segment)
|
||||
}
|
||||
// A trailing `**` would match files, directories, and symlinks below the
|
||||
// prefix; the walker models `**` only as a directory-spanning segment, so
|
||||
// it must reject the form loudly instead of silently returning nothing.
|
||||
if (segments[segments.length - 1] === '**') {
|
||||
throw new TypeError(`repo-files walker does not model a trailing ** segment in pattern: ${pattern}`)
|
||||
}
|
||||
const out: string[] = []
|
||||
|
||||
const visit = (dirAbs: string, dirRel: string, index: number): void => {
|
||||
if (index >= segments.length) return
|
||||
if (segments[index] !== '**') {
|
||||
visitSegment(dirAbs, dirRel, index)
|
||||
return
|
||||
}
|
||||
// `**` consumes zero directories here and one directory per recursion.
|
||||
visitSegment(dirAbs, dirRel, index + 1)
|
||||
for (const entry of readdirSync(dirAbs, { withFileTypes: true })) {
|
||||
if (entry.name.startsWith('.') || !entry.isDirectory()) continue
|
||||
visit(join(dirAbs, entry.name), dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`, index)
|
||||
}
|
||||
}
|
||||
|
||||
const visitSegment = (dirAbs: string, dirRel: string, index: number): void => {
|
||||
if (index >= segments.length) return
|
||||
const segment = segments[index]
|
||||
if (segment === undefined) return
|
||||
if (segment === '**') {
|
||||
visit(dirAbs, dirRel, index)
|
||||
return
|
||||
}
|
||||
const last = index === segments.length - 1
|
||||
for (const entry of readdirSync(dirAbs, { withFileTypes: true })) {
|
||||
if (!segmentMatches(segment, entry.name)) continue
|
||||
const childAbs = join(dirAbs, entry.name)
|
||||
const childRel = dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`
|
||||
if (last) {
|
||||
// Files and symlinks match; a broken or cyclic symlink then fails
|
||||
// loudly in the caller's realpathSync exactly as node glob did,
|
||||
// rather than silently shrinking the scanned corpus.
|
||||
if (entry.isFile() || entry.isSymbolicLink()) out.push(childRel)
|
||||
continue
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
visitSegment(childAbs, childRel, index + 1)
|
||||
continue
|
||||
}
|
||||
// A literal non-final segment resolves through stat like node glob's,
|
||||
// so it enters a symlinked directory; wildcard segments never reach
|
||||
// this branch because they are matched from dirent types above.
|
||||
if (!entry.isSymbolicLink() || hasWildcard(segment)) continue
|
||||
let target: ReturnType<typeof statSync>
|
||||
try {
|
||||
target = statSync(childAbs)
|
||||
} catch {
|
||||
// A broken symlink matches nothing under a literal segment, as node
|
||||
// glob silently returns no match for it.
|
||||
continue
|
||||
}
|
||||
if (!target.isDirectory()) continue
|
||||
visitSegment(childAbs, childRel, index + 1)
|
||||
}
|
||||
}
|
||||
|
||||
visit(root, '.', 0)
|
||||
return out.sort()
|
||||
}
|
||||
|
||||
/** Whether a pattern segment contains `*` or `?` and is therefore wildcard. */
|
||||
function hasWildcard(segment: string): boolean {
|
||||
return segment.includes('*') || segment.includes('?')
|
||||
}
|
||||
|
||||
/**
|
||||
* Expand repository-relative globs and deduplicate symlinked files.
|
||||
* @param root - absolute repository root.
|
||||
@@ -41,8 +208,7 @@ export function uniqueRepoFiles(
|
||||
const seen = new Set<string>()
|
||||
const files: RepoFile[] = []
|
||||
for (const pattern of patterns) {
|
||||
for (const match of globSync(pattern, { cwd: root })) {
|
||||
const repoPath = match.split(sep).join('/')
|
||||
for (const repoPath of expandGlob(root, pattern)) {
|
||||
if (isExcluded(repoPath)) continue
|
||||
const abs = resolve(root, repoPath)
|
||||
const real = realpathSync(abs)
|
||||
|
||||
Reference in new issue
Block a user