fix(code-language): address review on the persisted hint and aliases

Review threads: shrink READ_LANG_BY_EXTENSION to the suffixes whose own name is the better label (tsx/jsx/tf/tfvars/gradle) since the other 41 values already equal the language short name; label .hcl as hcl and .tfvars as tfvars instead of tf; accept .groovy alongside .gradle with its own short name; document that LANG_ALIASES also carries the persisted short ids; route the three inline alias lookups through grammarForHint; state that a short name may equal the grammar id; fix the util README wording.

Tests: cover .groovy, the four self-naming suffixes, and every id each suffix reaches through the shipped surfaces (the hand-synced LAZY_ALIASES list could not catch an alias whose target is neither boot nor lazy).
This commit is contained in:
mektpoy committed 2026-09-23 19:57:19 +08:00
1 parent c5c09c9afa
commit 2a0531e9c3
10 files changed
+93 -58

No files matched your search

@@ -36,8 +36,8 @@ type LangModule = { default: typeof langTs }
* resolve to the TypeScript grammar rather than a separate one: it tokenizes
* plain TS/JS exactly, and JSX/TSX approximately (shiki's TS grammar is not the
* dedicated TSX grammar, so JSX elements tokenize imperfectly) — an accepted
* trade to keep the boot set to one JS-family grammar. The read card's wider
* set loads lazily through {@link LAZY_GRAMMARS}.
* trade to keep the boot set to one JS-family grammar. Every other language in
* the shared extension table loads lazily through {@link LAZY_GRAMMARS}.
*/
const LANGS = [langTs, langBash, langJson]
@@ -115,9 +115,10 @@ const LAZY_GRAMMARS = new Map<string, () => Promise<LangModule>>([
* plain. A Map, not an object: fence info strings are assistant-authored, so
* a label like `constructor` or `__proto__` must miss instead of resolving an
* inherited property and crashing the renderer inside shiki. Keys cover both
* the markdown-fence aliases `CodeBlock` uses and the file-extension language
* ids `@deepseek-ai/dsh-util-code-language` resolves, so both callers resolve the
* same grammars. The JS family maps to the TypeScript grammar (see {@link LANGS} for
* the markdown-fence aliases `CodeBlock` uses, the file-extension language ids
* `@deepseek-ai/dsh-util-code-language` resolves, and the short ids
* `readLangHintForPath` persists, so every caller resolves the same grammars.
* A new short name in the shared table must be aliased here too. The JS family maps to the TypeScript grammar (see {@link LANGS} for
* the JSX/TSX approximation). A value not in {@link LANGS} names a
* {@link LAZY_GRAMMARS} entry loaded on first use.
*/
@@ -209,6 +210,9 @@ const LANG_ALIASES = new Map<string, string>([
['pl', 'perl'],
['pm', 'perl'],
['verilog', 'verilog'],
// Shiki's own `v` grammar is the V language; the fence label and the `.v`
// extension both name Verilog here, so a future V registration needs a
// different alias.
['v', 'verilog'],
['system-verilog', 'system-verilog'],
['systemverilog', 'system-verilog'],
@@ -379,7 +383,7 @@ const warmupTimer = setTimeout(() => { highlighter() }, 0)
* @returns the highlighted HTML, or `undefined` for unknown or not-yet-loaded languages.
*/
export function highlightToHtml(code: string, lang: string | undefined): string | undefined {
const resolved = lang === undefined ? undefined : LANG_ALIASES.get(lang.toLowerCase())
const resolved = grammarForHint(lang)
if (resolved === undefined) return undefined
if (!ensureGrammar(resolved)) return undefined
return highlighter().codeToHtml(code, { lang: resolved, theme: 'css-variables' })
@@ -505,7 +509,7 @@ export class StreamingHighlightSession {
this.lastCode = code
this.lastLang = lang
this.lastResult = undefined
const resolved = lang === undefined ? undefined : LANG_ALIASES.get(lang.toLowerCase())
const resolved = grammarForHint(lang)
if (resolved === undefined || !ensureGrammar(resolved)) {
this.reset(undefined)
return undefined
@@ -579,7 +583,7 @@ export interface StreamingHighlightFrame {
* @returns one entry per source line (each an array of runs), or `undefined` for unknown or not-yet-loaded languages.
*/
export function highlightLines(code: string, lang: string | undefined): HighlightSpan[][] | undefined {
const resolved = lang === undefined ? undefined : LANG_ALIASES.get(lang.toLowerCase())
const resolved = grammarForHint(lang)
if (resolved === undefined) return undefined
if (!ensureGrammar(resolved)) return undefined
const { tokens } = highlighter().codeToTokens(code, { lang: resolved, theme: 'css-variables' })
@@ -4,6 +4,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { act, cleanup, fireEvent, render, screen } from '@testing-library/react'
import type { ComponentProps } from 'react'
import { CodeBlock as LocalizedCodeBlock } from '../src/markdown/CodeBlock.tsx'
import { CODE_HIGHLIGHT_EXTENSIONS, languageForPath } from '../src/code-highlighting.ts'
import { readLangHintForPath } from '@deepseek-ai/dsh-util-code-language'
import { highlightToHtml, subscribeGrammarLoaded } from '../src/markdown/highlight.ts'
import { markdownLabels } from './labels.client.ts'
@@ -62,6 +64,32 @@ describe('highlightToHtml', () => {
}
// 57 dynamic grammars (some with large embedded sub-grammars) exceed the default.
}, 120_000)
it('highlights both ids every suffix reaches through the shipped surfaces', async () => {
// LAZY_ALIASES above is a hand-synced list, so it cannot catch an alias whose
// target is in neither LANGS nor LAZY_GRAMMARS: ensureGrammar treats that as a
// registered boot grammar, supportsHighlighting still reports true, and the
// render then throws. Walking the shared table's two ids per suffix covers
// every hint the Code preview and the read card can actually produce.
const hints = new Set<string>()
for (const extension of CODE_HIGHLIGHT_EXTENSIONS) {
const path = `file.${extension}`
for (const hint of [languageForPath(path), readLangHintForPath(path)]) if (hint !== undefined) hints.add(hint)
}
expect(hints.size).toBeGreaterThan(50)
const allLoaded = (): boolean => [...hints].every(hint => highlightToHtml('x', hint) !== undefined)
const registered = Promise.withResolvers<undefined>()
const stop = subscribeGrammarLoaded(() => { if (allLoaded()) registered.resolve(undefined) })
try {
// Touching each hint starts its dynamic import. A hint already loaded by an
// earlier test fires no further notification, so only await when needed.
for (const hint of hints) highlightToHtml('x', hint)
if (!allLoaded()) await registered.promise
for (const hint of hints) expect(highlightToHtml('x', hint), hint).toContain('shiki')
} finally {
stop()
}
}, 120_000)
})
describe('CodeBlock', () => {
+2 -2
View File
@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write packages/util/README.md
README.md: bd89353f66f8e852f531b393135aff76bbd612ca
README.zh.md: 52807d86206317a31fe5196825bf8a0ac5402d62
README.md: b5fca2657a5cde6a3de870153e74919ba36f032d
README.zh.md: dfe7d0621915f00804222b2690408916ce27a4bf
+1 -1
View File
@@ -39,7 +39,7 @@ Each package provides one primitive; open a package page for how to use it.
| [`lazy-require/`](lazy-require/README.md) | Loads CommonJS-compatible Host dependencies on first use with caller-relative resolution |
| [`native-command/`](native-command/README.md) | Runs host-native commands directly, never through a shell string |
| [`workspace-path/`](workspace-path/README.md) | Provides browser-safe Workspace path and display helpers |
| [`code-language/`](code-language/README.md) | Maps a filename suffix to the syntax-highlighting language Code preview, diff review, and the read card render |
| [`code-language/`](code-language/README.md) | Maps a filename suffix to the syntax-highlighting language that the Code preview, diff review, and the read card render |
| [`output-retention/`](output-retention/README.md) | Bounds model-facing output and reports exact omission metadata |
| [`time/`](time/README.md) | Validates and canonicalizes a caller-reported IANA time zone |
| [`timeout/`](timeout/README.md) | Deadline arithmetic, signal fusion, and timeout-versus-cancel classification |
+1 -1
View File
@@ -39,7 +39,7 @@ kind: "package-group"
| [`lazy-require/`](lazy-require/README.zh.md) | 在首次使用时按调用方位置加载兼容 CommonJS 的 Host 依赖 |
| [`native-command/`](native-command/README.zh.md) | 直接运行宿主原生命令,绝不拼 shell 字符串 |
| [`workspace-path/`](workspace-path/README.zh.md) | 提供浏览器安全的 Workspace 路径与显示辅助函数 |
| [`code-language/`](code-language/README.zh.md) | 把文件名后缀映射为 Code 预览、diff 审阅与 read 卡片渲染的语法高亮语言 |
| [`code-language/`](code-language/README.zh.md) | 把文件名后缀映射为 Code 预览、diff 审阅与 read 卡片所渲染的语法高亮语言 |
| [`output-retention/`](output-retention/README.zh.md) | 限制面向模型的输出并报告精确的省略元数据 |
| [`time/`](time/README.zh.md) | 校验并规范化调用方所报的 IANA 时区 |
| [`timeout/`](timeout/README.zh.md) | 截止时间运算、信号融合与超时和取消分类 |
+2 -2
View File
@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write packages/util/code-language/README.md
README.md: 11883bcd6520bed1a8ca3a1043a18f8fde46ebe3
README.zh.md: 55de7a1046620ac6003bcefde27d28e1a8050b1f
README.md: 99930c192b946da09506593275e35dbd2601ddf4
README.zh.md: 5227ca85dadbd540c923ff94db375cdf80bffb52
+1 -1
View File
@@ -24,7 +24,7 @@ The repository's one file-extension to syntax-highlighting language table, share
The table lives in [`src/index.ts`](src/index.ts). Each key is a canonical language id — the grammar id the Client highlighter resolves — and each value is the lowercase extensions without the dot that select it. `languageForPath(path)` takes the text after the last dot of the final path segment, lowercases it, and looks it up in a `Map`; a `Map` keeps a filename such as `foo.constructor` from resolving an `Object.prototype` member. An unlisted dotfile (`.gitignore`), an absent extension, a trailing dot, and an unknown suffix all return `undefined`, which every consumer renders as plain text; a leading dot is still the separator, so `.env` resolves to `dotenv`. Both `/` and `\` separate path segments, so a Windows path resolves like a POSIX one.
The set is curated for common source, config, script, data, and markup extensions rather than mirroring a full language registry. Extensions with no matching grammar map to the nearest one (`properties` to `ini`, whose registration carries the `properties` alias); certificate and lock extensions (`pem`, `crt`, `key`, `cer`, `lock`) stay unlisted, and `csv` stays unlisted because the Spreadsheet preview declares the suffix and must keep it, so the earlier Code registration cannot claim it. `readLangHintForPath` is a projection over this table: a suffix whose value a recorded session already holds returns that persisted short id, any other suffix returns its language's short name (`powershell` to `ps1`, `hcl` to `tf`), and an unrecognized suffix returns `undefined` — so the persisted field holds one style, a short id, never a canonical grammar id. A consumer that needs the Client highlighter to actually tokenize a language still depends on that grammar being registered there — an id without a loaded grammar renders as plain text rather than failing.
The set is curated for common source, config, script, data, and markup extensions rather than mirroring a full language registry. Extensions with no matching grammar map to the nearest one (`properties` to `ini`, whose registration carries the `properties` alias); certificate and lock extensions (`pem`, `crt`, `key`, `cer`, `lock`) stay unlisted, and `csv` stays unlisted because the Spreadsheet preview declares the suffix and must keep it, so the earlier Code registration cannot claim it. `readLangHintForPath` is a projection over this table: a suffix whose value a recorded session already holds returns that persisted short id, any other suffix returns its language's short name (`powershell` to `ps1`), and an unrecognized suffix returns `undefined` — so the persisted field holds one style, a short name; for `kotlin`, `swift`, `yaml`, `json`, and similar that name is also the grammar id, and the suffixes whose own name is the better label (`tsx`, `tf`, `tfvars`, `gradle`) keep it. A consumer that needs the Client highlighter to actually tokenize a language still depends on that grammar being registered there — an id without a loaded grammar renders as plain text rather than failing.
-----
+1 -1
View File
@@ -24,7 +24,7 @@ kind: "package-library"
表位于 [`src/index.ts`](src/index.ts)。每个键是规范化语言 id——即 Client 高亮器解析的 grammar id;每个值是选中它的、不含点的小写扩展名。`languageForPath(path)` 取路径最后一段的最后一个点之后的文本、转小写后在 `Map` 中查找;使用 `Map` 可避免 `foo.constructor` 这类文件名解析到 `Object.prototype` 成员。未列入表的 dotfile(`.gitignore`)、无扩展名、结尾点、未知后缀都返回 `undefined`,各消费方一律按纯文本渲染;前导点仍算分隔符,因此 `.env` 解析为 `dotenv`。`/` 与 `\` 都算路径分隔符,因此 Windows 路径与 POSIX 路径解析一致。
该集合按常见源码、配置、脚本、数据与标记扩展名精选,并非完整语言登记表。没有对应 grammar 的扩展名映射到最接近的 grammar(`properties` 映射到 `ini`,该 grammar 自带 `properties` alias);证书与锁文件扩展名(`pem`、`crt`、`key`、`cer`、`lock`)保持不表,`csv` 同样保持不表:Spreadsheet 预览声明该后缀且必须保留,因此更早注册的 Code 不能抢占它。`readLangHintForPath` 在同一张表上做投影:已录制的会话已经持有该后缀的值时返回该持久化短 id,其余后缀返回该语言的短名(`powershell`→`ps1`、`hcl`→`tf`),无法识别的后缀返回 `undefined`——持久化字段因此只有一种风格,即短 id,绝不出现规范 grammar id。消费方若希望 Client 高亮器真正分词,仍取决于该 grammar 已在其中注册——没有加载 grammar 的 id 会按纯文本渲染,而不是报错。
该集合按常见源码、配置、脚本、数据与标记扩展名精选,并非完整语言登记表。没有对应 grammar 的扩展名映射到最接近的 grammar(`properties` 映射到 `ini`,该 grammar 自带 `properties` alias);证书与锁文件扩展名(`pem`、`crt`、`key`、`cer`、`lock`)保持不表,`csv` 同样保持不表:Spreadsheet 预览声明该后缀且必须保留,因此更早注册的 Code 不能抢占它。`readLangHintForPath` 在同一张表上做投影:已录制的会话已经持有该后缀的值时返回该持久化短 id,其余后缀返回该语言的短名(`powershell`→`ps1`),无法识别的后缀返回 `undefined`——持久化字段因此只有一种风格,即短名;对 `kotlin`、`swift`、`yaml`、`json` 等语言,短名与 grammar id 相同;而 `tsx`、`tf`、`tfvars`、`gradle` 这些后缀本身的名字更准确,保留其自身名字。消费方若希望 Client 高亮器真正分词,仍取决于该 grammar 已在其中注册——没有加载 grammar 的 id 会按纯文本渲染,而不是报错。
-----
+28 -31
View File
@@ -6,9 +6,10 @@
* `highlightLines`/`highlightToHtml` unchanged; a filename outside the table, or
* one naming a language the highlighter does not register, renders as plain text.
* The read tool persists a short `lang` id, so {@link readLangHintForPath}
* projects short ids over this table: a suffix whose value a recorded session
* already holds keeps it byte-identical, and every other suffix persists the
* canonical language's short name rather than its grammar id.
* projects short ids over this table: a suffix keeps the value a recorded session
* already holds, and every other suffix persists the language's short name —
* which for `kotlin`, `swift`, `yaml`, `json`, and similar equals that
* language's grammar id.
* @module @deepseek-ai/dsh-util-code-language
*/
@@ -23,7 +24,8 @@
* stay unlisted. Certificate and lock extensions (`pem`, `crt`, `key`, `cer`,
* `lock`) stay unlisted, and `csv` is deliberately absent: the Spreadsheet
* preview declares it and must keep it, so listing it here would let the earlier
* Code registration take the suffix.
* Code registration take the suffix. `makefile` covers only the suffix
* (`foo.makefile`); the extensionless `Makefile` name stays unlisted.
*/
const LANGUAGE_EXTENSIONS: Readonly<Record<string, readonly string[]>> = {
typescript: ['ts', 'tsx', 'mts', 'cts'],
@@ -85,7 +87,7 @@ const LANGUAGE_EXTENSIONS: Readonly<Record<string, readonly string[]>> = {
svelte: ['svelte'],
make: ['makefile', 'mk'],
cmake: ['cmake'],
groovy: ['gradle'],
groovy: ['gradle', 'groovy'],
}
/**
@@ -101,26 +103,20 @@ const LANGUAGES = new Map(Object.entries(LANGUAGE_EXTENSIONS)
export const CODE_HIGHLIGHT_EXTENSIONS: readonly string[] = [...LANGUAGES.keys()]
/**
* The short `lang` values recorded sessions already hold, keyed by the suffix
* that produced them. Keys are shared-table extensions, so the recognized set
* derives from {@link LANGUAGE_EXTENSIONS}; each value is the exact string a
* recorded session persists and must not change. An extension absent here but
* present in the shared table falls back to {@link SHORT_BY_LANGUAGE}, so every
* persisted hint is a short id.
* Persisted `lang` values a language-level short id cannot express, keyed by the
* suffix that produced them. Everything else falls back to
* {@link SHORT_BY_LANGUAGE}, so only a suffix whose own name is the better label
* appears here. Each value is the exact string recorded sessions already hold
* and must not change; the byte-level expectation for every suffix lives in the
* read consumer's test.
*/
const READ_LANG_BY_EXTENSION = new Map<string, string>(Object.entries({
ts: 'ts', tsx: 'tsx', mts: 'ts', cts: 'ts',
js: 'js', jsx: 'jsx', mjs: 'js', cjs: 'js',
json: 'json', jsonc: 'json',
py: 'py', rb: 'rb', go: 'go', rs: 'rs', java: 'java',
c: 'c', h: 'c', cc: 'cpp', cpp: 'cpp', hpp: 'cpp', cxx: 'cpp',
cs: 'cs', kt: 'kotlin', swift: 'swift', php: 'php',
sh: 'sh', bash: 'sh', zsh: 'sh',
yaml: 'yaml', yml: 'yaml', toml: 'toml', ini: 'ini',
md: 'md', markdown: 'md', mdx: 'mdx',
html: 'html', htm: 'html', css: 'css', scss: 'scss', less: 'less',
sql: 'sql', xml: 'xml', lua: 'lua',
}))
const READ_LANG_BY_EXTENSION = new Map<string, string>([
// The two JSX flavors keep their own suffix; every other persisted value is
// the language's short name, or an extension naming itself better than its
// language does (`tf` rather than `hcl`, `gradle` rather than `groovy`).
['tsx', 'tsx'], ['jsx', 'jsx'],
['tf', 'tf'], ['tfvars', 'tfvars'], ['gradle', 'gradle'],
])
/**
* The short `lang` id the read card persists, keyed by canonical language id.
@@ -185,13 +181,13 @@ const SHORT_BY_LANGUAGE: Readonly<Record<string, string>> = {
'system-verilog': 'sv',
graphql: 'graphql',
proto: 'proto',
hcl: 'tf',
hcl: 'hcl',
nix: 'nix',
vue: 'vue',
svelte: 'svelte',
make: 'make',
cmake: 'cmake',
groovy: 'gradle',
groovy: 'groovy',
}
function extensionForPath(path: string): string | undefined {
@@ -218,11 +214,12 @@ export function languageForPath(path: string): string | undefined {
/**
* Derive the Host read card's persisted `lang` hint from a read path's
* extension. The hint is always a short id: a suffix a recorded session already
* holds keeps its persisted value (`ts`, `md`, `cpp`, …), while a suffix with no
* recorded value (`.ps1`, `.env`, `.tf`, …) uses its language's short name
* (`ps1`, `env`, `tf`, …) rather than the canonical grammar id. An unrecognized
* suffix stays `undefined`, so the card renders as plain text.
* extension. The value is the language's short name; for some languages that name
* is also the grammar id (`kotlin`, `swift`, `java`, `yaml`, `json`), and
* {@link READ_LANG_BY_EXTENSION} overrides it where the suffix names itself
* better (`tsx`, `tf`, `gradle`). A suffix a recorded session already holds keeps
* its persisted value. An unrecognized suffix stays `undefined`, so the card
* renders as plain text.
* @param path - the model-facing path the read reported.
* @returns the persisted language hint, or `undefined` when the extension maps to none.
*/
@@ -24,7 +24,7 @@ describe('languageForPath', () => {
['Form.vb', 'vb'], ['script.pl', 'perl'], ['Module.pm', 'perl'],
['top.v', 'verilog'], ['top.sv', 'system-verilog'], ['defs.svh', 'system-verilog'],
['schema.graphql', 'graphql'], ['query.gql', 'graphql'], ['message.proto', 'proto'],
['main.tf', 'hcl'], ['terraform.tfvars', 'hcl'], ['stack.hcl', 'hcl'],
['main.tf', 'hcl'], ['terraform.tfvars', 'hcl'], ['stack.hcl', 'hcl'], ['build.groovy', 'groovy'],
['flake.nix', 'nix'], ['App.vue', 'vue'], ['App.svelte', 'svelte'],
['Makefile', undefined], ['build.mk', 'make'], ['CMakeLists.cmake', 'cmake'],
['build.gradle', 'groovy'],
@@ -117,6 +117,13 @@ describe('readLangHintForPath', () => {
expect(readLangHintForPath('paper.tex')).toBe('tex')
expect(readLangHintForPath('server.log')).toBe('log')
expect(readLangHintForPath('message.proto')).toBe('proto')
// `.hcl`/`.tf`/`.tfvars` and `.gradle`/`.groovy` keep the label their own
// suffix names, so a Nomad `.hcl` file is never labelled `tf`.
expect(readLangHintForPath('nomad.hcl')).toBe('hcl')
expect(readLangHintForPath('main.tf')).toBe('tf')
expect(readLangHintForPath('terraform.tfvars')).toBe('tfvars')
expect(readLangHintForPath('build.gradle')).toBe('gradle')
expect(readLangHintForPath('build.groovy')).toBe('groovy')
})
// The short id each canonical language persists. Transcribed rather than
@@ -132,18 +139,17 @@ describe('readLangHintForPath', () => {
xml: 'xml', lua: 'lua', bat: 'bat', powershell: 'ps1', r: 'r', julia: 'jl',
dart: 'dart', scala: 'scala', clojure: 'clj', erlang: 'erl', elixir: 'ex',
haskell: 'hs', fsharp: 'fs', vb: 'vb', perl: 'pl', verilog: 'v',
'system-verilog': 'sv', graphql: 'graphql', proto: 'proto', hcl: 'tf', nix: 'nix',
vue: 'vue', svelte: 'svelte', make: 'make', cmake: 'cmake', groovy: 'gradle',
'system-verilog': 'sv', graphql: 'graphql', proto: 'proto', hcl: 'hcl', nix: 'nix',
vue: 'vue', svelte: 'svelte', make: 'make', cmake: 'cmake', groovy: 'groovy',
}
it('persists a short id for every suffix, never a canonical grammar id', () => {
// The persisted `lang` has one style: a short id. Every suffix must resolve
// to one of these short names, except the persisted `tsx`/`jsx` values,
// which the language table cannot express because typescript and javascript
// already shorten to `ts` and `js`. A result outside both sets is a
// canonical grammar id leaking back in (`powershell`, `dotenv`, `latex`,
// `hcl`).
const allowed = new Set([...Object.values(SHORT_BY_LANGUAGE), 'tsx', 'jsx'])
it('persists a short id for every suffix', () => {
// The persisted `lang` is the language's short name, plus the suffixes whose
// own name is the better label. These languages' short names happen to equal
// their grammar ids (`kotlin`, `swift`, `yaml`), so a result outside both
// sets means a canonical id leaked back in for a language that has a
// distinct short name (`powershell`, `dotenv`, `latex`, `hcl`).
const allowed = new Set([...Object.values(SHORT_BY_LANGUAGE), 'tsx', 'jsx', 'tf', 'tfvars', 'gradle'])
for (const extension of CODE_HIGHLIGHT_EXTENSIONS) {
const hint = readLangHintForPath(`file.${extension}`)
expect(hint, extension).toBeDefined()