diff --git a/scripts/repo-files.spec.ts b/scripts/repo-files.spec.ts new file mode 100644 index 0000000000..60875341d7 --- /dev/null +++ b/scripts/repo-files.spec.ts @@ -0,0 +1,176 @@ +import { mkdtempSync, mkdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, relative } from 'node:path' +import { describe, expect, it } from 'vitest' +import { uniqueRepoFiles } from './repo-files.ts' + +interface Tree { + root: string + clean: () => void +} + +function makeTree(): Tree { + const parent = mkdtempSync(join(tmpdir(), 'repo-files-')) + const root = join(parent, 'root') + const outside = join(parent, 'outside') // reachable only through a symlinked dir + mkdirSync(join(root, 'a'), { recursive: true }) + writeFileSync(join(root, 'a', 'snap.md'), 'real\n') + mkdirSync(join(root, 'd'), { recursive: true }) + symlinkSync(join(root, 'a', 'snap.md'), join(root, 'd', 'snap.md')) + mkdirSync(outside, { recursive: true }) + writeFileSync(join(outside, 'snap.md'), 'behind a symlinked dir\n') + symlinkSync(outside, join(root, 'linked-dir')) + mkdirSync(join(root, '.hidden'), { recursive: true }) + writeFileSync(join(root, '.hidden', 'snap.md'), 'hidden by dot\n') + return { root, clean: () => { rmSync(parent, { recursive: true, force: true }) } } +} + +describe('uniqueRepoFiles', () => { + it('enumerates ** matches without probing a symlinked file as a directory', () => { + const tree = makeTree() + try { + // Node's internal glob (from some 24.x releases) lstat-probes + // /snap.md while expanding `**/snap.md` and throws ENOTDIR. + // The walker must return the real files, deduplicated by canonical + // target, without throwing on any node version. + const files = uniqueRepoFiles(tree.root, ['**/snap.md']) + const rootReal = realpathSync(tree.root) + const reals = files.map(file => relative(rootReal, file.real)).sort() + expect(reals).toEqual([join('a', 'snap.md')]) + } finally { + tree.clean() + } + }) + + it('does not follow symlinked directories under ** or wildcard-match dot names', () => { + const tree = makeTree() + try { + const files = uniqueRepoFiles(tree.root, ['**/*.md']) + const paths = files.map(file => relative(tree.root, file.abs)).sort() + // The symlinked d/snap.md dedupes onto its a/snap.md target; the + // linked-dir and .hidden targets must not appear at all. + expect(paths).toEqual([join('a', 'snap.md')]) + } finally { + tree.clean() + } + }) + + it('follows a literal segment that names a symlinked directory', () => { + const tree = makeTree() + try { + // Node glob resolves literal segments with stat, so `linked-dir/**` and + // `linked-dir/*` enter the symlinked directory's target; `**` and + // wildcard segments resolve with dirent types and do not. + const files = uniqueRepoFiles(tree.root, ['linked-dir/**/*.md']) + expect(files.map(file => relative(tree.root, file.abs))).toEqual([join('linked-dir', 'snap.md')]) + expect(uniqueRepoFiles(tree.root, ['linked-dir/*.md']).map(file => relative(tree.root, file.abs))) + .toEqual([join('linked-dir', 'snap.md')]) + // A wildcard first segment never enters the symlinked directory. + expect(uniqueRepoFiles(tree.root, ['*/snap.md']).map(file => relative(tree.root, file.abs))) + .toEqual([join('a', 'snap.md')]) + } finally { + tree.clean() + } + }) + + it('follows repeated literal symlinked directories like node glob and terminates', () => { + const parent = mkdtempSync(join(tmpdir(), 'repo-files-cycle-')) + try { + const root = join(parent, 'root') + mkdirSync(join(root, 'a'), { recursive: true }) + writeFileSync(join(root, 'a', 'snap.md'), 'x\n') + symlinkSync(root, join(root, 'cyc')) + // Each literal `cyc` segment resolves through stat and enters the + // symlinked directory again, exactly as node glob does for a repeated + // literal; recursion stays bounded because each literal consumes one + // pattern segment and `**` only enters real directories. + expect(uniqueRepoFiles(root, ['cyc/**/snap.md']).map(file => relative(root, file.abs))) + .toEqual([join('cyc', 'a', 'snap.md')]) + expect(uniqueRepoFiles(root, ['cyc/cyc/**/snap.md']).map(file => relative(root, file.abs))) + .toEqual([join('cyc', 'cyc', 'a', 'snap.md')]) + expect(uniqueRepoFiles(root, ['cyc/cyc/cyc/**/snap.md']).map(file => relative(root, file.abs))) + .toEqual([join('cyc', 'cyc', 'cyc', 'a', 'snap.md')]) + } finally { + rmSync(parent, { recursive: true, force: true }) + } + }) + + it('reports a matched file canonical target for downstream existence checks', () => { + const tree = makeTree() + try { + const files = uniqueRepoFiles(tree.root, ['a/*.md']) + expect(files).toHaveLength(1) + expect(files[0]!.real).toBe(join(realpathSync(tree.root), 'a', 'snap.md')) + } finally { + tree.clean() + } + }) + + it('fails loudly on a broken symlink instead of shrinking the corpus', () => { + const tree = makeTree() + try { + // A healthy tree with symlinked files must not throw. + expect(() => uniqueRepoFiles(tree.root, ['a/*.md'])).not.toThrow() + } finally { + tree.clean() + } + const parent = mkdtempSync(join(tmpdir(), 'repo-files-broken-')) + try { + const root = join(parent, 'root') + mkdirSync(root, { recursive: true }) + writeFileSync(join(root, 'gone.md'), 'x\n') + symlinkSync(join(root, 'gone.md'), join(root, 'broken-link.md')) + rmSync(join(root, 'gone.md')) + // realpathSync on the matched broken link must throw, exactly as it did + // under the node-glob implementation, instead of silently shrinking the + // scanned corpus. + expect(() => uniqueRepoFiles(root, ['*.md'])).toThrow() + // A broken symlink under a literal non-final segment matches nothing, + // again as node glob silently returns no match for it. + symlinkSync(join(root, 'gone-dir'), join(root, 'broken-dir')) + expect(uniqueRepoFiles(root, ['broken-dir/*.md'])).toEqual([]) + } finally { + rmSync(parent, { recursive: true, force: true }) + } + }) + + it('rejects glob syntax the walker does not model instead of matching nothing', () => { + const tree = makeTree() + try { + // Node glob would interpret the bracket class; the walker must fail + // loudly rather than expand the pattern as a literal and quietly match + // nothing. + expect(() => uniqueRepoFiles(tree.root, ['**/*.[cm]d'])).toThrow(/does not model glob syntax/) + } finally { + tree.clean() + } + }) + + it('rejects a trailing ** segment instead of silently returning nothing', () => { + const tree = makeTree() + try { + expect(() => uniqueRepoFiles(tree.root, ['a/**'])).toThrow(/trailing \*\*/) + // A trailing slash yields an empty final segment that can never match. + expect(() => uniqueRepoFiles(tree.root, ['a/**/'])).toThrow(/empty segments/) + } finally { + tree.clean() + } + }) + + it('folds a . segment and rejects a .. segment like node glob semantics', () => { + const tree = makeTree() + try { + // Node glob normalizes a `.` segment away, so `./a/snap.md` matches + // a/snap.md instead of looking for a directory named `.`. + expect(uniqueRepoFiles(tree.root, ['./a/snap.md']).map(file => relative(tree.root, file.abs))) + .toEqual([join('a', 'snap.md')]) + expect(uniqueRepoFiles(tree.root, ['a/./snap.md']).map(file => relative(tree.root, file.abs))) + .toEqual([join('a', 'snap.md')]) + // A `..` segment escapes the scanned root; the walker must fail loudly + // rather than silently match nothing. + expect(() => uniqueRepoFiles(tree.root, ['a/../snap.md'])).toThrow(/does not model \.\. segments/) + } finally { + tree.clean() + } + }) +}) diff --git a/scripts/repo-files.ts b/scripts/repo-files.ts index 7c7aca276d..56d9cbf9f7 100644 --- a/scripts/repo-files.ts +++ b/scripts/repo-files.ts @@ -1,7 +1,7 @@ /** Shared repository file discovery and line-oriented reference scanning. */ -import { globSync, readFileSync, realpathSync } from 'node:fs' -import { relative, resolve, sep } from 'node:path' +import { readdirSync, readFileSync, realpathSync, statSync } from 'node:fs' +import { join, relative, resolve, sep } from 'node:path' /** One authored path plus its canonical target for symlink deduplication. */ export interface RepoFile { @@ -26,6 +26,173 @@ export function isArchivedAgentNotePath(path: string): boolean { return path.replaceAll('\\', '/').startsWith('.agents/notes/archived/') } +/** + * Whether a pattern segment matches a directory or file name. Supports `*` and + * `?` inside a segment and mirrors node's glob `dot: false`: a segment whose + * first character is a wildcard does not match dot names. `**` is handled as a + * whole segment by the walker, never here. + */ +function segmentMatches(pattern: string, name: string): boolean { + if (name.startsWith('.') && (pattern.startsWith('*') || pattern.startsWith('?'))) return false + let expression = '' + for (const character of pattern) { + if (character === '*') expression += '.*' + else if (character === '?') expression += '.' + else expression += character.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + } + return new RegExp(`^${expression}$`).test(name) +} + +/** + * Reject glob syntax the walker does not model. Node's `fs.glob` understands + * character classes, brace alternation, and extglobs; expanding those + * silently as literals would match nothing and quietly shrink a gate's + * corpus, so a pattern segment using them fails loudly instead. Pure literal + * segments must therefore contain none of the rejected metacharacters + * either, even where node glob would read them literally. + */ +const UNSUPPORTED_GLOB = /[[\]{}()!+@\\]/ + +function assertSupportedSegment(segment: string): void { + if (UNSUPPORTED_GLOB.test(segment)) { + throw new TypeError(`repo-files walker does not model glob syntax in segment: ${segment}`) + } +} + +/** + * Walk `root` matching one repository-relative glob without node's `fs.glob`. + * The repository's own walker exists because node's internal glob, from some + * 24.x releases, lstat-probes `/` for symlinked files + * while expanding `**` and throws ENOTDIR instead of skipping (observed on + * node 24.13.0 scanning `snapshots/acp/image-compaction`'s symlinked + * `system-prompt.expected.md`). The walker decides directoryhood from dirent + * types and stat results, never by probing a path under a file, so the same + * tree enumerates identically on every node version. + * + * Segment semantics: literal segments resolve through `stat`, so a literal + * naming a symlinked directory enters that directory exactly as node glob + * does — including a literal repeated across segments, which node glob + * resolves each time; wildcard segments and `**` resolve through dirent + * types, so they never enter symlinked directories and never match dot + * names. Node documents `follow: false` only for `**` expansion, so the + * wildcard-side behavior is this walker's own contract, pinned by + * repo-files.spec.ts, rather than a cross-version node guarantee; the four + * consuming gates' current patterns contain no wildcard segment over a + * symlinked directory, so the corpus is unchanged. `**` spans zero or more + * directories. The final segment matches files and symlinks; a broken or + * cyclic symlink then fails loudly in the caller's realpathSync rather than + * shrinking the scanned corpus, while a broken symlink under a literal + * non-final segment matches nothing, again as node glob does. Traversal + * terminates without a visited set: each literal segment consumes one pattern + * segment per recursion, and `**` recurses only into real directories, which + * form a finite tree because symlinked directories are never expanded by it. + * Pattern segments support `*`, `?`, and literals that contain none of + * `[]{}()!+@\`; other node-glob syntax, including a trailing `**`, `..`, and + * empty segments, is rejected loudly up front, while a `.` segment is folded + * away exactly as node glob normalizes it. Returns repository-relative slash + * paths, sorted. + */ +function expandGlob(root: string, pattern: string): string[] { + const segments: string[] = [] + for (const segment of pattern.split('/')) { + if (segment === '.') { + // Node glob normalizes a `.` segment away; dropping it here keeps + // `./README.md` and `a/./b` matching exactly as node glob does instead + // of silently matching nothing. + continue + } + if (segment === '..') { + // A `..` segment escapes the scanned root and interacts with `**` in + // ways node glob special-cases; no gate pattern uses one, so the walker + // rejects the form loudly instead of silently matching nothing. + throw new TypeError(`repo-files walker does not model .. segments in pattern: ${pattern}`) + } + if (segment === '') { + // A leading, doubled, or trailing slash yields an empty segment that + // can never match an entry; node glob would tolerate the form, so the + // walker must reject it loudly instead of silently returning nothing. + throw new TypeError(`repo-files walker does not model empty segments in pattern: ${pattern}`) + } + segments.push(segment) + } + if (segments.length === 0) { + throw new TypeError(`repo-files walker does not model a pattern with no segments: ${pattern}`) + } + for (const segment of segments) { + if (segment !== '**') assertSupportedSegment(segment) + } + // A trailing `**` would match files, directories, and symlinks below the + // prefix; the walker models `**` only as a directory-spanning segment, so + // it must reject the form loudly instead of silently returning nothing. + if (segments[segments.length - 1] === '**') { + throw new TypeError(`repo-files walker does not model a trailing ** segment in pattern: ${pattern}`) + } + const out: string[] = [] + + const visit = (dirAbs: string, dirRel: string, index: number): void => { + if (index >= segments.length) return + if (segments[index] !== '**') { + visitSegment(dirAbs, dirRel, index) + return + } + // `**` consumes zero directories here and one directory per recursion. + visitSegment(dirAbs, dirRel, index + 1) + for (const entry of readdirSync(dirAbs, { withFileTypes: true })) { + if (entry.name.startsWith('.') || !entry.isDirectory()) continue + visit(join(dirAbs, entry.name), dirRel === '.' ? entry.name : `${dirRel}/${entry.name}`, index) + } + } + + const visitSegment = (dirAbs: string, dirRel: string, index: number): void => { + if (index >= segments.length) return + const segment = segments[index] + if (segment === undefined) return + if (segment === '**') { + visit(dirAbs, dirRel, index) + return + } + const last = index === segments.length - 1 + for (const entry of readdirSync(dirAbs, { withFileTypes: true })) { + if (!segmentMatches(segment, entry.name)) continue + const childAbs = join(dirAbs, entry.name) + const childRel = dirRel === '.' ? entry.name : `${dirRel}/${entry.name}` + if (last) { + // Files and symlinks match; a broken or cyclic symlink then fails + // loudly in the caller's realpathSync exactly as node glob did, + // rather than silently shrinking the scanned corpus. + if (entry.isFile() || entry.isSymbolicLink()) out.push(childRel) + continue + } + if (entry.isDirectory()) { + visitSegment(childAbs, childRel, index + 1) + continue + } + // A literal non-final segment resolves through stat like node glob's, + // so it enters a symlinked directory; wildcard segments never reach + // this branch because they are matched from dirent types above. + if (!entry.isSymbolicLink() || hasWildcard(segment)) continue + let target: ReturnType + try { + target = statSync(childAbs) + } catch { + // A broken symlink matches nothing under a literal segment, as node + // glob silently returns no match for it. + continue + } + if (!target.isDirectory()) continue + visitSegment(childAbs, childRel, index + 1) + } + } + + visit(root, '.', 0) + return out.sort() +} + +/** Whether a pattern segment contains `*` or `?` and is therefore wildcard. */ +function hasWildcard(segment: string): boolean { + return segment.includes('*') || segment.includes('?') +} + /** * Expand repository-relative globs and deduplicate symlinked files. * @param root - absolute repository root. @@ -41,8 +208,7 @@ export function uniqueRepoFiles( const seen = new Set() const files: RepoFile[] = [] for (const pattern of patterns) { - for (const match of globSync(pattern, { cwd: root })) { - const repoPath = match.split(sep).join('/') + for (const repoPath of expandGlob(root, pattern)) { if (isExcluded(repoPath)) continue const abs = resolve(root, repoPath) const real = realpathSync(abs)