diff --git a/packages/client/ui-primitives/README.md b/packages/client/ui-primitives/README.md index 9ce34c7095..f9bb2a770a 100644 --- a/packages/client/ui-primitives/README.md +++ b/packages/client/ui-primitives/README.md @@ -199,6 +199,7 @@ These limits define how the atoms behave at the edges; they are current package - **Known-site marks are a fixed list** — only the named hosts resolve to their own mark, and every other external host keeps the globe; recognizing an arbitrary site would require fetching its icon over the network. - **Streaming defers cross-boundary reference resolution** — a reference-style link or footnote whose definition sits on the other side of the incremental freeze boundary renders as literal text while the reply streams; the settled full parse at finalize resolves it. - **A long highlighted fence retains its complete token DOM** — streaming avoids re-parsing, re-tokenizing, and reconciling the completed prefix, but it does not discard old colors or virtualize token spans. Final DOM cardinality therefore still follows the fence's token count; nested/container fences and a pathological single long line remain on the general tail path. +- **Overlong lines stay uncolored** — a source line of 1,000 or more UTF-16 units renders as one plain run in every highlighted surface, because TextMate scanning cost grows superlinearly with line length and a single generated line can otherwise block the page for seconds. Highlighting still runs synchronously during render, so a file with thousands of ordinary lines costs proportionally more. - **The fish logo is a redrawn approximation** — its font glyph has no exportable vector geometry in the local design data, so a hand-authored recreation stands in until an exact export path exists. - **`Pill` and `Input` have no design source** — both atoms are self-defined; the sidebar search field and view-tab strip that resemble them are consumer-owned compositions, not these atoms. - **No `Active` `StateDot` variant** — the supported states are done, warning, ongoing, error, and idle. diff --git a/packages/client/ui-primitives/src/markdown/highlight.ts b/packages/client/ui-primitives/src/markdown/highlight.ts index 0ae164d138..94a7b3a59a 100644 --- a/packages/client/ui-primitives/src/markdown/highlight.ts +++ b/packages/client/ui-primitives/src/markdown/highlight.ts @@ -275,6 +275,14 @@ const regexEngine = createJavaScriptRegexEngine({ }), }) +/** + * Lines at least this many UTF-16 units long render as one uncolored token. + * TextMate scanning cost grows superlinearly with line length (a 20,000-unit + * string literal blocks the main thread for seconds), and minified or + * generated lines are where that length occurs. + */ +const TOKENIZE_MAX_LINE_LENGTH = 1000 + let singleton: HighlighterCore | undefined /** Representative paths through every boot grammar, compiled before user content is timed. */ @@ -386,7 +394,9 @@ export function highlightToHtml(code: string, lang: string | undefined): string const resolved = grammarForHint(lang) if (resolved === undefined) return undefined if (!ensureGrammar(resolved)) return undefined - return highlighter().codeToHtml(code, { lang: resolved, theme: 'css-variables' }) + return highlighter().codeToHtml(code, { + lang: resolved, theme: 'css-variables', tokenizeMaxLineLength: TOKENIZE_MAX_LINE_LENGTH, + }) } /** @@ -492,6 +502,7 @@ export class StreamingHighlightSession { return highlighter().codeToTokensBase(text, { lang: resolved, theme: 'css-variables', + tokenizeMaxLineLength: TOKENIZE_MAX_LINE_LENGTH, ...(this.state === undefined ? {} : { grammarState: this.state }), }) } @@ -586,7 +597,9 @@ export function highlightLines(code: string, lang: string | undefined): Highligh const resolved = grammarForHint(lang) if (resolved === undefined) return undefined if (!ensureGrammar(resolved)) return undefined - const { tokens } = highlighter().codeToTokens(code, { lang: resolved, theme: 'css-variables' }) + const { tokens } = highlighter().codeToTokens(code, { + lang: resolved, theme: 'css-variables', tokenizeMaxLineLength: TOKENIZE_MAX_LINE_LENGTH, + }) // shiki tokenizes `a\nb` into two lines; a trailing newline (`a\n`) adds a // third, empty line the caller's own line array does not carry. Drop that // one terminator line so the two structures stay in step. The explicit diff --git a/packages/client/ui-primitives/tests/code-highlighting.client.spec.tsx b/packages/client/ui-primitives/tests/code-highlighting.client.spec.tsx index 1c2b829a5d..fdfa55a686 100644 --- a/packages/client/ui-primitives/tests/code-highlighting.client.spec.tsx +++ b/packages/client/ui-primitives/tests/code-highlighting.client.spec.tsx @@ -4,7 +4,9 @@ import { afterEach, describe, expect, it } from 'vitest' import { cleanup, renderHook, waitFor } from '@testing-library/react' import { languageForPath as sharedLanguageForPath, readLangHintForPath } from '@deepseek-ai/dsh-util-code-language' import { CODE_HIGHLIGHT_EXTENSIONS, languageForPath, useCodeHighlighter } from '../src/code-highlighting.ts' -import { grammarForHint, supportsHighlighting } from '../src/markdown/highlight.ts' +import { + StreamingHighlightSession, grammarForHint, highlightLines, highlightToHtml, supportsHighlighting, +} from '../src/markdown/highlight.ts' afterEach(cleanup) @@ -102,4 +104,15 @@ describe('code highlighting', () => { } expect(hook.result.current('sample')).not.toBeUndefined() }) + + it('leaves an overlong line as one uncolored run while its neighbors still highlight', () => { + const literal = `const table = "${'ab'.repeat(500)}"` + const code = ['const before = 1', literal, 'const after = 2'].join('\n') + const lines = highlightLines(code, 'javascript') + expect(lines?.map(line => line.length > 1)).toEqual([true, false, true]) + expect(lines?.[1]).toEqual([{ text: literal, style: { color: '' } }]) + expect(new StreamingHighlightSession().update(code, 'javascript')?.[1]).toEqual([{ text: literal, style: { color: '' } }]) + const html = highlightToHtml(code, 'javascript') + expect(html).toContain(`${literal}`) + }) })