From dc07e5a50dadcf03e3fb9e1b7c69bb2ba9550254 Mon Sep 17 00:00:00 2001 From: mektpoy Date: Mon, 21 Sep 2026 17:36:16 +0800 Subject: [PATCH] fix(tools): keep surrogate pairs intact when capping tool output Three tools capped the preview text they return with String.prototype.slice at a UTF-16 code-unit budget. A cut that lands inside a surrogate pair leaves a lone high surrogate, which is not well-formed text: a strict JSON reader rejects the durable Session log carrying it, and the preview shows a broken glyph. A lone surrogate retained in history has already blocked every later Messages request for one session. Centralize the cap in output-retention as truncateWithoutSplittingSurrogatePair, and use it from the persistent bash and pwsh shells and the string-replace editor. The cut drops the unpaired half instead of keeping it, so the retained text stays a prefix and is at most one code unit shorter than before. The new dependency edges and the shifted Config sources regenerate docs/module-graph and docs/config-catalog, including the committed translation pair. Refs #4817 --- docs/config-catalog.i18n.yaml | 4 +-- docs/config-catalog.md | 6 ++-- docs/config-catalog.zh.md | 6 ++-- docs/module-graph.i18n.yaml | 4 +-- docs/module-graph.md | 9 +++-- docs/module-graph.zh.md | 9 +++-- .../fs/tool-str-replace-editor/package.json | 2 ++ .../fs/tool-str-replace-editor/src/index.ts | 3 +- .../tests/tools.spec.ts | 15 +++++++++ .../fs/tool-str-replace-editor/tsconfig.json | 1 + .../shell/tool-bash-persistent/package.json | 2 ++ .../shell/tool-bash-persistent/src/index.ts | 3 +- .../tool-bash-persistent/tests/tools.spec.ts | 18 +++++++++- .../shell/tool-bash-persistent/tsconfig.json | 1 + .../shell/tool-pwsh-persistent/package.json | 2 ++ .../shell/tool-pwsh-persistent/src/index.ts | 3 +- .../tool-pwsh-persistent/tests/tools.spec.ts | 18 +++++++++- .../shell/tool-pwsh-persistent/tsconfig.json | 1 + .../util/output-retention/README.i18n.yaml | 4 +-- packages/util/output-retention/README.md | 4 +-- packages/util/output-retention/README.zh.md | 4 +-- packages/util/output-retention/src/index.ts | 17 ++++++++++ .../tests/output-retention.spec.ts | 33 +++++++++++++++++++ pnpm-lock.yaml | 9 +++++ 24 files changed, 151 insertions(+), 27 deletions(-) diff --git a/docs/config-catalog.i18n.yaml b/docs/config-catalog.i18n.yaml index 43f977e619..bbb66436dd 100644 --- a/docs/config-catalog.i18n.yaml +++ b/docs/config-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/config-catalog.md -config-catalog.md: c46008dd88e51dc079d72e6605d06176c26bdea1 -config-catalog.zh.md: 7376476771902e1bbcba3ae4d6374b1087e14a72 +config-catalog.md: 6aa27aec967066deb217d1a065b093ba4f09f20f +config-catalog.zh.md: 7051e686c6081c5da65745335bbda2ddef81edce diff --git a/docs/config-catalog.md b/docs/config-catalog.md index c46008dd88..6aa27aec96 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -3249,7 +3249,7 @@ export interface Config { } ``` -Source: [`packages/shell/tool-bash-persistent/src/index.ts:443`](../packages/shell/tool-bash-persistent/src/index.ts) +Source: [`packages/shell/tool-bash-persistent/src/index.ts:444`](../packages/shell/tool-bash-persistent/src/index.ts) @@ -3443,7 +3443,7 @@ export interface Config { } ``` -Source: [`packages/shell/tool-pwsh-persistent/src/index.ts:480`](../packages/shell/tool-pwsh-persistent/src/index.ts) +Source: [`packages/shell/tool-pwsh-persistent/src/index.ts:481`](../packages/shell/tool-pwsh-persistent/src/index.ts) @@ -3517,7 +3517,7 @@ export interface Config { } ``` -Source: [`packages/fs/tool-str-replace-editor/src/index.ts:505`](../packages/fs/tool-str-replace-editor/src/index.ts) +Source: [`packages/fs/tool-str-replace-editor/src/index.ts:506`](../packages/fs/tool-str-replace-editor/src/index.ts) diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 7376476771..7051e686c6 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -3251,7 +3251,7 @@ export interface Config { } ``` -来源: [`packages/shell/tool-bash-persistent/src/index.ts:443`](../packages/shell/tool-bash-persistent/src/index.ts) +来源: [`packages/shell/tool-bash-persistent/src/index.ts:444`](../packages/shell/tool-bash-persistent/src/index.ts) @@ -3445,7 +3445,7 @@ export interface Config { } ``` -来源: [`packages/shell/tool-pwsh-persistent/src/index.ts:480`](../packages/shell/tool-pwsh-persistent/src/index.ts) +来源: [`packages/shell/tool-pwsh-persistent/src/index.ts:481`](../packages/shell/tool-pwsh-persistent/src/index.ts) @@ -3519,7 +3519,7 @@ export interface Config { } ``` -来源: [`packages/fs/tool-str-replace-editor/src/index.ts:505`](../packages/fs/tool-str-replace-editor/src/index.ts) +来源: [`packages/fs/tool-str-replace-editor/src/index.ts:506`](../packages/fs/tool-str-replace-editor/src/index.ts) diff --git a/docs/module-graph.i18n.yaml b/docs/module-graph.i18n.yaml index 235cd3fdef..12cbb59624 100644 --- a/docs/module-graph.i18n.yaml +++ b/docs/module-graph.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/module-graph.md -module-graph.md: e36487734f39545fee9b188d9443336ad7ed2e0e -module-graph.zh.md: 646aa21ea348ddf5e73d12b94247319cfe3768d6 +module-graph.md: 44e499d26af5753a260d6bc0802ef67adb6de475 +module-graph.zh.md: 8e95ddc3a53956c92194c3274484695dcf599619 diff --git a/docs/module-graph.md b/docs/module-graph.md index e36487734f..44e499d26a 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -780,6 +780,7 @@ flowchart TD pkg_tool_fs_search --> pkg_timeout pkg_tool_fs_search --> pkg_tools pkg_tool_str_replace_editor --> pkg_fs + pkg_tool_str_replace_editor --> pkg_output_retention pkg_tool_str_replace_editor --> pkg_sandbox pkg_tool_str_replace_editor --> pkg_sandbox_policy pkg_tool_str_replace_editor --> pkg_tools @@ -929,10 +930,12 @@ flowchart TD pkg_shell_env --> pkg_shell pkg_shell_env --> pkg_tools pkg_tool_bash_persistent --> pkg_agent + pkg_tool_bash_persistent --> pkg_output_retention pkg_tool_bash_persistent --> pkg_terminal pkg_tool_bash_persistent --> pkg_timeout pkg_tool_bash_persistent --> pkg_tools pkg_tool_pwsh_persistent --> pkg_agent + pkg_tool_pwsh_persistent --> pkg_output_retention pkg_tool_pwsh_persistent --> pkg_terminal pkg_tool_pwsh_persistent --> pkg_timeout pkg_tool_pwsh_persistent --> pkg_tools @@ -1540,7 +1543,7 @@ flowchart TD | [`tool-goal`](../packages/goal/tool-goal) | `goal` | [`agent`](../packages/core/agent), [`goal`](../packages/goal/goal), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | | [`tool-fs`](../packages/fs/tool-fs) | `fs` | [`attachment`](../packages/attachment/attachment), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | [`tool-fs-search`](../packages/fs/tool-fs-search) | `fs` | [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`session`](../packages/core/session), [`spill`](../packages/spill/spill), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | -| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) | +| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`output-retention`](../packages/util/output-retention), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) | | [`tool-skill`](../packages/skill/tool-skill) | `skill` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`skill`](../packages/skill/skill), [`tools`](../packages/core/tools) | | [`tool-workspace-dependencies`](../packages/skill/tool-workspace-dependencies) | `skill` | [`tools`](../packages/core/tools) | | [`tool-web`](../packages/web/tool-web) | `web` | [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`web`](../packages/web/web) | @@ -1572,8 +1575,8 @@ flowchart TD | [`session-title-all-prompts-llm`](../packages/session/session-title-all-prompts-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) | | [`session-title-first-prompt-llm`](../packages/session/session-title-first-prompt-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) | | [`shell-env`](../packages/shell/shell-env) | `shell` | [`home-paths`](../packages/util/home-paths), [`shell`](../packages/shell/shell), [`tools`](../packages/core/tools) | -| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | -| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | +| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | +| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | | [`ssh`](../packages/ssh/ssh) | `ssh` | [`brand`](../packages/util/brand), [`fs`](../packages/fs/fs), [`fs-local`](../packages/fs/fs-local), [`fs-sandbox`](../packages/fs/fs-sandbox), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-local`](../packages/sandbox/sandbox-local), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session-projection`](../packages/session/session-projection), [`subprocess`](../packages/subprocess/subprocess), [`subprocess-local`](../packages/subprocess/subprocess-local) | | [`tool-terminal`](../packages/terminal/tool-terminal) | `terminal` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`system-prompt`](../packages/core/system-prompt), [`terminal`](../packages/terminal/terminal), [`tools`](../packages/core/tools) | | [`llm-replay`](../packages/test-support/llm-replay) | `test-support` | [`compaction`](../packages/compaction/compaction), [`deepseek-llm-api-extensions`](../packages/llm/deepseek-llm-api-extensions), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | diff --git a/docs/module-graph.zh.md b/docs/module-graph.zh.md index 646aa21ea3..8e95ddc3a5 100644 --- a/docs/module-graph.zh.md +++ b/docs/module-graph.zh.md @@ -782,6 +782,7 @@ flowchart TD pkg_tool_fs_search --> pkg_timeout pkg_tool_fs_search --> pkg_tools pkg_tool_str_replace_editor --> pkg_fs + pkg_tool_str_replace_editor --> pkg_output_retention pkg_tool_str_replace_editor --> pkg_sandbox pkg_tool_str_replace_editor --> pkg_sandbox_policy pkg_tool_str_replace_editor --> pkg_tools @@ -931,10 +932,12 @@ flowchart TD pkg_shell_env --> pkg_shell pkg_shell_env --> pkg_tools pkg_tool_bash_persistent --> pkg_agent + pkg_tool_bash_persistent --> pkg_output_retention pkg_tool_bash_persistent --> pkg_terminal pkg_tool_bash_persistent --> pkg_timeout pkg_tool_bash_persistent --> pkg_tools pkg_tool_pwsh_persistent --> pkg_agent + pkg_tool_pwsh_persistent --> pkg_output_retention pkg_tool_pwsh_persistent --> pkg_terminal pkg_tool_pwsh_persistent --> pkg_timeout pkg_tool_pwsh_persistent --> pkg_tools @@ -1542,7 +1545,7 @@ flowchart TD | [`tool-goal`](../packages/goal/tool-goal) | `goal` | [`agent`](../packages/core/agent), [`goal`](../packages/goal/goal), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | | [`tool-fs`](../packages/fs/tool-fs) | `fs` | [`attachment`](../packages/attachment/attachment), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | [`tool-fs-search`](../packages/fs/tool-fs-search) | `fs` | [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`session`](../packages/core/session), [`spill`](../packages/spill/spill), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | -| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) | +| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`output-retention`](../packages/util/output-retention), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) | | [`tool-skill`](../packages/skill/tool-skill) | `skill` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`skill`](../packages/skill/skill), [`tools`](../packages/core/tools) | | [`tool-workspace-dependencies`](../packages/skill/tool-workspace-dependencies) | `skill` | [`tools`](../packages/core/tools) | | [`tool-web`](../packages/web/tool-web) | `web` | [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`web`](../packages/web/web) | @@ -1574,8 +1577,8 @@ flowchart TD | [`session-title-all-prompts-llm`](../packages/session/session-title-all-prompts-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) | | [`session-title-first-prompt-llm`](../packages/session/session-title-first-prompt-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) | | [`shell-env`](../packages/shell/shell-env) | `shell` | [`home-paths`](../packages/util/home-paths), [`shell`](../packages/shell/shell), [`tools`](../packages/core/tools) | -| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | -| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | +| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | +| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) | | [`ssh`](../packages/ssh/ssh) | `ssh` | [`brand`](../packages/util/brand), [`fs`](../packages/fs/fs), [`fs-local`](../packages/fs/fs-local), [`fs-sandbox`](../packages/fs/fs-sandbox), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-local`](../packages/sandbox/sandbox-local), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session-projection`](../packages/session/session-projection), [`subprocess`](../packages/subprocess/subprocess), [`subprocess-local`](../packages/subprocess/subprocess-local) | | [`tool-terminal`](../packages/terminal/tool-terminal) | `terminal` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`system-prompt`](../packages/core/system-prompt), [`terminal`](../packages/terminal/terminal), [`tools`](../packages/core/tools) | | [`llm-replay`](../packages/test-support/llm-replay) | `test-support` | [`compaction`](../packages/compaction/compaction), [`deepseek-llm-api-extensions`](../packages/llm/deepseek-llm-api-extensions), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | diff --git a/packages/fs/tool-str-replace-editor/package.json b/packages/fs/tool-str-replace-editor/package.json index da5a7bc917..4119e284f9 100644 --- a/packages/fs/tool-str-replace-editor/package.json +++ b/packages/fs/tool-str-replace-editor/package.json @@ -27,6 +27,7 @@ "license": "MIT", "peerDependencies": { "@deepseek-ai/dsh-fs": "workspace:^", + "@deepseek-ai/dsh-output-retention": "workspace:^", "@deepseek-ai/dsh-sandbox": "workspace:^", "@deepseek-ai/dsh-sandbox-policy": "workspace:^", "@deepseek-ai/dsh-tools": "workspace:^", @@ -43,6 +44,7 @@ "@deepseek-ai/dsh-fs-observation-policy": "workspace:^", "@deepseek-ai/dsh-fs-sandbox": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-output-retention": "workspace:^", "@deepseek-ai/dsh-sandbox": "workspace:^", "@deepseek-ai/dsh-sandbox-policy": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", diff --git a/packages/fs/tool-str-replace-editor/src/index.ts b/packages/fs/tool-str-replace-editor/src/index.ts index f77736685e..3542ee1770 100644 --- a/packages/fs/tool-str-replace-editor/src/index.ts +++ b/packages/fs/tool-str-replace-editor/src/index.ts @@ -8,6 +8,7 @@ import type { Context } from '@deepseek-ai/cordis' import z from '@deepseek-ai/schemastery' import { FsError } from '@deepseek-ai/dsh-fs' import type { FsInfo, FsTarget, FsWriteIntent } from '@deepseek-ai/dsh-fs' +import { truncateWithoutSplittingSurrogatePair } from '@deepseek-ai/dsh-output-retention' import { sandboxDenialMarker } from '@deepseek-ai/dsh-sandbox' import type { SandboxExecutionPolicy } from '@deepseek-ai/dsh-sandbox' import type { SandboxPolicyService } from '@deepseek-ai/dsh-sandbox-policy' @@ -33,7 +34,7 @@ Notes for using the \`str_replace\` command: function maybeTruncate(content: string, maxOutputChars: number): string { return content.length <= maxOutputChars ? content - : content.slice(0, maxOutputChars) + TRUNCATED_MESSAGE + : truncateWithoutSplittingSurrogatePair(content, maxOutputChars) + TRUNCATED_MESSAGE } function codepointCompare(left: string, right: string): number { diff --git a/packages/fs/tool-str-replace-editor/tests/tools.spec.ts b/packages/fs/tool-str-replace-editor/tests/tools.spec.ts index 1f893f37fe..2bd7633706 100644 --- a/packages/fs/tool-str-replace-editor/tests/tools.spec.ts +++ b/packages/fs/tool-str-replace-editor/tests/tools.spec.ts @@ -347,6 +347,21 @@ describe('tool-str-replace-editor', () => { .toContain('') }) + it('clips a view at a code-unit boundary without splitting a surrogate pair', async () => { + const wide = await setup({ maxOutputChars: 10_000 }) + const file = join(wide.root, 'emoji.txt') + await writeFile(file, `${'😀'.repeat(4)}tail`) + const untruncated = text(await call(wide.ctx, wide.owner, { command: 'view', path: file })) + // The cap lands on the high surrogate of the first emoji. + const splitAt = untruncated.indexOf('😀') + 1 + + const clipped = await setup({ maxOutputChars: splitAt }) + const rendered = text(await call(clipped.ctx, clipped.owner, { command: 'view', path: file })) + + expect(rendered.startsWith(`${untruncated.slice(0, splitAt - 1)}`)).toBe(true) + expect(rendered).not.toContain('\uD83D') + }) + it('matches canonical empty-line, range, and end-insert behavior', async () => { const { ctx, root, owner } = await setup() const empty = join(root, 'empty.txt') diff --git a/packages/fs/tool-str-replace-editor/tsconfig.json b/packages/fs/tool-str-replace-editor/tsconfig.json index b9dba81d5d..686cded3e0 100644 --- a/packages/fs/tool-str-replace-editor/tsconfig.json +++ b/packages/fs/tool-str-replace-editor/tsconfig.json @@ -8,6 +8,7 @@ "references": [ { "path": "../../../vendor/cordis" }, { "path": "../../../vendor/schemastery" }, + { "path": "../../util/output-retention" }, { "path": "../../core/tools" }, { "path": "../fs" }, { "path": "../../sandbox/sandbox" }, diff --git a/packages/shell/tool-bash-persistent/package.json b/packages/shell/tool-bash-persistent/package.json index 7587e0be20..f3f65c754a 100644 --- a/packages/shell/tool-bash-persistent/package.json +++ b/packages/shell/tool-bash-persistent/package.json @@ -27,6 +27,7 @@ "license": "MIT", "peerDependencies": { "@deepseek-ai/dsh-agent": "workspace:^", + "@deepseek-ai/dsh-output-retention": "workspace:^", "@deepseek-ai/dsh-terminal": "workspace:^", "@deepseek-ai/dsh-timeout": "workspace:^", "@deepseek-ai/dsh-tools": "workspace:^", @@ -41,6 +42,7 @@ "@deepseek-ai/dsh-agent": "workspace:^", "@deepseek-ai/dsh-agent-loop-testkit": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-output-retention": "workspace:^", "@deepseek-ai/dsh-terminal": "workspace:^", "@deepseek-ai/dsh-terminal-bash": "workspace:^", "@deepseek-ai/dsh-sandbox": "workspace:^", diff --git a/packages/shell/tool-bash-persistent/src/index.ts b/packages/shell/tool-bash-persistent/src/index.ts index c248ca94b5..ff6aa0f5be 100644 --- a/packages/shell/tool-bash-persistent/src/index.ts +++ b/packages/shell/tool-bash-persistent/src/index.ts @@ -7,6 +7,7 @@ import { randomUUID } from 'node:crypto' import type { Context } from '@deepseek-ai/cordis' import z from '@deepseek-ai/schemastery' import type { Agent } from '@deepseek-ai/dsh-agent' +import { truncateWithoutSplittingSurrogatePair } from '@deepseek-ai/dsh-output-retention' import type { TerminalReadResult, TerminalSessionId } from '@deepseek-ai/dsh-terminal' import { deadline, timeoutOf } from '@deepseek-ai/dsh-timeout' import { defineTool } from '@deepseek-ai/dsh-tools' @@ -58,7 +59,7 @@ function maybeTruncate(content: string, maxOutputChars: number, incomplete = fal if (content.length <= maxOutputChars && !incomplete) return content return content.length <= maxOutputChars ? content + TRUNCATED_MESSAGE - : content.slice(0, maxOutputChars) + TRUNCATED_MESSAGE + : truncateWithoutSplittingSurrogatePair(content, maxOutputChars) + TRUNCATED_MESSAGE } function markers(): CommandMarkers { diff --git a/packages/shell/tool-bash-persistent/tests/tools.spec.ts b/packages/shell/tool-bash-persistent/tests/tools.spec.ts index a79d116556..cc962c4447 100644 --- a/packages/shell/tool-bash-persistent/tests/tools.spec.ts +++ b/packages/shell/tool-bash-persistent/tests/tools.spec.ts @@ -114,6 +114,7 @@ class StubPtySession implements TerminalBackendSession { pendingText = '' historyTruncated = false throwOnSend = false + largeOutput = 'x'.repeat(100) constructor(mode: StubMode) { this.mode = mode @@ -206,7 +207,7 @@ class StubPtySession implements TerminalBackendSession { return this.operation(Promise.resolve(this.result(output, 'stdin_read'))) } const commandOutput = this.mode === 'large' - ? 'x'.repeat(100) + ? this.largeOutput : this.mode === 'nonzero' ? '' : 'hello from stub' const exitCode = this.mode === 'nonzero' ? 7 : 0 const output = `${start ?? ''}\n${commandOutput}\n${end ?? ''}${exitCode}\n${this.motd}` @@ -404,6 +405,21 @@ describe('tool-bash-persistent', () => { expect(stub.sessions[2]?.closed).toEqual(['external cleanup']) }) + it('clips a split surrogate pair instead of leaving a lone half', async () => { + const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 10 }) + await call(ctx, owner, 'warm up') + const session = stub.sessions[0]! + session.mode = 'large' + // The tenth code unit is the emoji's high surrogate, so the clip drops the + // unpaired half instead of emitting it. + session.largeOutput = `${'x'.repeat(9)}😀tail` + + const rendered = text(await call(ctx, owner, 'emoji')) + + expect(rendered.startsWith(`${'x'.repeat(9)}`)).toBe(true) + expect(rendered).not.toContain('\uD83D') + }) + it('waits for status digits after a torn completion marker', async () => { const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 1_000 }) await call(ctx, owner, 'warm up') diff --git a/packages/shell/tool-bash-persistent/tsconfig.json b/packages/shell/tool-bash-persistent/tsconfig.json index fe64fdde1f..e3b72276d1 100644 --- a/packages/shell/tool-bash-persistent/tsconfig.json +++ b/packages/shell/tool-bash-persistent/tsconfig.json @@ -8,6 +8,7 @@ "references": [ { "path": "../../../vendor/cordis" }, { "path": "../../../vendor/schemastery" }, + { "path": "../../util/output-retention" }, { "path": "../../core/agent" }, { "path": "../../core/tools" }, { "path": "../../terminal/terminal" }, diff --git a/packages/shell/tool-pwsh-persistent/package.json b/packages/shell/tool-pwsh-persistent/package.json index 2d14f4f879..a1147eaf54 100644 --- a/packages/shell/tool-pwsh-persistent/package.json +++ b/packages/shell/tool-pwsh-persistent/package.json @@ -27,6 +27,7 @@ "license": "MIT", "peerDependencies": { "@deepseek-ai/dsh-agent": "workspace:^", + "@deepseek-ai/dsh-output-retention": "workspace:^", "@deepseek-ai/dsh-terminal": "workspace:^", "@deepseek-ai/dsh-timeout": "workspace:^", "@deepseek-ai/dsh-tools": "workspace:^", @@ -41,6 +42,7 @@ "@deepseek-ai/dsh-agent": "workspace:^", "@deepseek-ai/dsh-agent-loop-testkit": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-output-retention": "workspace:^", "@deepseek-ai/dsh-terminal": "workspace:^", "@deepseek-ai/dsh-terminal-bash": "workspace:^", "@deepseek-ai/dsh-pwsh-local": "workspace:^", diff --git a/packages/shell/tool-pwsh-persistent/src/index.ts b/packages/shell/tool-pwsh-persistent/src/index.ts index 34f8cf7732..0448ddc076 100644 --- a/packages/shell/tool-pwsh-persistent/src/index.ts +++ b/packages/shell/tool-pwsh-persistent/src/index.ts @@ -9,6 +9,7 @@ import { randomUUID } from 'node:crypto' import type { Context } from '@deepseek-ai/cordis' import z from '@deepseek-ai/schemastery' import type { Agent } from '@deepseek-ai/dsh-agent' +import { truncateWithoutSplittingSurrogatePair } from '@deepseek-ai/dsh-output-retention' import type { TerminalReadResult, TerminalSendResult, TerminalSessionId } from '@deepseek-ai/dsh-terminal' import { deadline, timeoutOf } from '@deepseek-ai/dsh-timeout' import { defineTool } from '@deepseek-ai/dsh-tools' @@ -58,7 +59,7 @@ function maybeTruncate(content: string, maxOutputChars: number, incomplete = fal if (content.length <= maxOutputChars && !incomplete) return content return content.length <= maxOutputChars ? content + TRUNCATED_MESSAGE - : content.slice(0, maxOutputChars) + TRUNCATED_MESSAGE + : truncateWithoutSplittingSurrogatePair(content, maxOutputChars) + TRUNCATED_MESSAGE } function markers(): CommandMarkers { diff --git a/packages/shell/tool-pwsh-persistent/tests/tools.spec.ts b/packages/shell/tool-pwsh-persistent/tests/tools.spec.ts index c260f818eb..67825d8b17 100644 --- a/packages/shell/tool-pwsh-persistent/tests/tools.spec.ts +++ b/packages/shell/tool-pwsh-persistent/tests/tools.spec.ts @@ -119,6 +119,7 @@ class StubTerminalSession implements TerminalBackendSession { pendingText = '' historyTruncated = false throwOnSend = false + largeOutput = 'x'.repeat(100) constructor(mode: StubMode) { this.mode = mode @@ -218,7 +219,7 @@ class StubTerminalSession implements TerminalBackendSession { return this.operation(Promise.resolve(this.result(output, 'stdin_read'))) } const commandOutput = this.mode === 'large' - ? 'x'.repeat(100) + ? this.largeOutput : this.mode === 'nonzero' ? '' : this.mode === 'prompt-collision' ? this.motd : 'hello from stub' @@ -455,6 +456,21 @@ describe('tool-pwsh-persistent', () => { expect(stub.sessions[2]?.closed).toEqual(['external cleanup']) }) + it('clips a split surrogate pair instead of leaving a lone half', async () => { + const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 10 }) + await call(ctx, owner, 'warm up') + const session = stub.sessions[0]! + session.mode = 'large' + // The tenth code unit is the emoji's high surrogate, so the clip drops the + // unpaired half instead of emitting it. + session.largeOutput = `${'x'.repeat(9)}😀tail` + + const rendered = text(await call(ctx, owner, 'emoji')) + + expect(rendered.startsWith(`${'x'.repeat(9)}`)).toBe(true) + expect(rendered).not.toContain('\uD83D') + }) + it('waits for status digits after a torn completion marker', async () => { const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 1_000 }) await call(ctx, owner, 'warm up') diff --git a/packages/shell/tool-pwsh-persistent/tsconfig.json b/packages/shell/tool-pwsh-persistent/tsconfig.json index fe64fdde1f..e3b72276d1 100644 --- a/packages/shell/tool-pwsh-persistent/tsconfig.json +++ b/packages/shell/tool-pwsh-persistent/tsconfig.json @@ -8,6 +8,7 @@ "references": [ { "path": "../../../vendor/cordis" }, { "path": "../../../vendor/schemastery" }, + { "path": "../../util/output-retention" }, { "path": "../../core/agent" }, { "path": "../../core/tools" }, { "path": "../../terminal/terminal" }, diff --git a/packages/util/output-retention/README.i18n.yaml b/packages/util/output-retention/README.i18n.yaml index 71736fe9bf..53c073815a 100644 --- a/packages/util/output-retention/README.i18n.yaml +++ b/packages/util/output-retention/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/util/output-retention/README.md -README.md: bb0902b959f298f9f7b0a64de23d48e29bfd75ad -README.zh.md: 7d96ba4da33a98b24eba53f9dd5a2f2c98b79f39 +README.md: ec16dd9626f2268a9a019373c3e135619e1ef2e5 +README.zh.md: 1ae37399224838d6b044dacc81a15983f82ff9c8 diff --git a/packages/util/output-retention/README.md b/packages/util/output-retention/README.md index bb0902b959..ec16dd9626 100644 --- a/packages/util/output-retention/README.md +++ b/packages/util/output-retention/README.md @@ -9,7 +9,7 @@ English | [中文](README.zh.md) ## Summary -Use `dsh-output-retention` to cap the items or text a tool returns to a model while reporting what was omitted. `ItemRetainer` keeps an ordered head window and can report an exact omitted-item count; `TextRetainer` keeps head, tail, or head-and-tail byte windows without returning invalid UTF-8 cuts. `formatRetentionNotice` adds a consistent omission clause while each tool supplies its own recovery guidance. Grouping, line numbering, spill files, and provider errors remain tool responsibilities; consumers import this library directly rather than loading it through `cordis.yml`. +Use `dsh-output-retention` to cap the items or text a tool returns to a model while reporting what was omitted. `ItemRetainer` keeps an ordered head window and can report an exact omitted-item count; `TextRetainer` keeps head, tail, or head-and-tail byte windows without returning invalid UTF-8 cuts. `formatRetentionNotice` adds a consistent omission clause while each tool supplies its own recovery guidance. `truncateWithoutSplittingSurrogatePair` caps a character-budget preview without leaving a lone high surrogate at the cut. Grouping, line numbering, spill files, and provider errors remain tool responsibilities; consumers import this library directly rather than loading it through `cordis.yml`. ## Table of Contents @@ -104,7 +104,7 @@ The library is built on one separation: it owns the mechanical question of what | File | Role | |---|---| -| [`src/index.ts`](src/index.ts) | `ItemRetainer`, `TextRetainer`, `describeOmitted`, and `formatRetentionNotice` | +| [`src/index.ts`](src/index.ts) | `ItemRetainer`, `TextRetainer`, `describeOmitted`, `formatRetentionNotice`, and `truncateWithoutSplittingSurrogatePair` | | — | No runtime invariant companion is published; this pure utility owns no event stream or mutable runtime data; its value algebra is enforced by unit tests. | ### Two retainers, two resource models diff --git a/packages/util/output-retention/README.zh.md b/packages/util/output-retention/README.zh.md index 7d96ba4da3..1ae3739922 100644 --- a/packages/util/output-retention/README.zh.md +++ b/packages/util/output-retention/README.zh.md @@ -9,7 +9,7 @@ kind: "package-library" ## 概述 -使用 `dsh-output-retention` 限制工具返回给模型的项或文本量,并报告省略了什么。`ItemRetainer` 保留有序的头部窗口,并可报告精确的省略项数;`TextRetainer` 保留 head、tail 或 head-and-tail 字节窗口,且不会返回因切割而无效的 UTF-8。`formatRetentionNotice` 添加一致的省略子句,各工具则提供自己的恢复指引。分组、行号、spill 文件与提供方错误仍归工具负责;消费方直接导入本库,而不通过 `cordis.yml` 加载。 +使用 `dsh-output-retention` 限制工具返回给模型的项或文本量,并报告省略了什么。`ItemRetainer` 保留有序的头部窗口,并可报告精确的省略项数;`TextRetainer` 保留 head、tail 或 head-and-tail 字节窗口,且不会返回因切割而无效的 UTF-8。`formatRetentionNotice` 添加一致的省略子句,各工具则提供自己的恢复指引。`truncateWithoutSplittingSurrogatePair` 按字符预算截断预览时,不会在切口留下孤立的高位代理项。分组、行号、spill 文件与提供方错误仍归工具负责;消费方直接导入本库,而不通过 `cordis.yml` 加载。 ## 目录 @@ -104,7 +104,7 @@ const footer = formatRetentionNotice( | 文件 | 职责 | |---|---| -| [`src/index.ts`](src/index.ts) | `ItemRetainer`、`TextRetainer`、`describeOmitted` 与 `formatRetentionNotice` | +| [`src/index.ts`](src/index.ts) | `ItemRetainer`、`TextRetainer`、`describeOmitted`、`formatRetentionNotice` 与 `truncateWithoutSplittingSurrogatePair` | | — | 不发布运行时不变式伴生入口;这个纯工具不拥有事件流或可变运行时数据;其值代数由单元测试保证。 | ### 两个 retainer,两种资源模型 diff --git a/packages/util/output-retention/src/index.ts b/packages/util/output-retention/src/index.ts index 5091c16374..3c2ba40e43 100644 --- a/packages/util/output-retention/src/index.ts +++ b/packages/util/output-retention/src/index.ts @@ -441,3 +441,20 @@ export function formatRetentionNotice( .filter(part => part.length > 0) .join(' ') } + +/** + * Cap `text` at `maxChars` UTF-16 code units without ending in a lone high + * surrogate: a cut that lands inside a surrogate pair drops the unpaired half, + * so the result is one code unit shorter than the cap. An unpaired surrogate is + * not well-formed text — a strict JSON reader rejects a durable Session log that + * carries one. + * + * @param text The text to cap. + * @param maxChars The maximum number of UTF-16 code units to retain. + * @returns A prefix of `text` no longer than `maxChars`, never ending in a lone high surrogate. + */ +export function truncateWithoutSplittingSurrogatePair(text: string, maxChars: number): string { + if (text.length <= maxChars) return text + const capped = text.slice(0, maxChars) + return /[\uD800-\uDBFF]$/.test(capped) ? capped.slice(0, -1) : capped +} diff --git a/packages/util/output-retention/tests/output-retention.spec.ts b/packages/util/output-retention/tests/output-retention.spec.ts index dfad8eb251..e1f5c93d92 100644 --- a/packages/util/output-retention/tests/output-retention.spec.ts +++ b/packages/util/output-retention/tests/output-retention.spec.ts @@ -6,6 +6,7 @@ import { type Omitted, type RetentionNotice, TextRetainer, + truncateWithoutSplittingSurrogatePair, } from '@deepseek-ai/dsh-output-retention' /** Decode a RetainedText via a round-trip helper for readable UTF-8 assertions. */ @@ -374,3 +375,35 @@ describe('formatRetentionNotice', () => { expect(out).toBe('Omitted 500 bytes. Kept 2000B head + 2000B tail.') }) }) + +describe('truncateWithoutSplittingSurrogatePair', () => { + it('caps at maxChars UTF-16 code units', () => { + expect(truncateWithoutSplittingSurrogatePair('abcdef', 3)).toBe('abc') + expect(truncateWithoutSplittingSurrogatePair('abcdef', 0)).toBe('') + expect(truncateWithoutSplittingSurrogatePair('abcdef', 1)).toBe('a') + }) + + it('returns text within the cap unchanged', () => { + expect(truncateWithoutSplittingSurrogatePair('abc', 3)).toBe('abc') + expect(truncateWithoutSplittingSurrogatePair('abc', 9)).toBe('abc') + expect(truncateWithoutSplittingSurrogatePair('', 0)).toBe('') + }) + + it('drops the unpaired half when the cap lands inside a surrogate pair', () => { + // '😀' is two code units, so these caps would otherwise keep its high surrogate alone. + expect(truncateWithoutSplittingSurrogatePair('ab😀cd', 3)).toBe('ab') + expect(truncateWithoutSplittingSurrogatePair('ab😀cd', 4)).toBe('ab😀') + expect(truncateWithoutSplittingSurrogatePair('😀😀', 1)).toBe('') + expect(truncateWithoutSplittingSurrogatePair('😀😀', 3)).toBe('😀') + }) + + it('never ends in a lone high surrogate at any cap', () => { + const text = 'a😀b😀c' + for (let maxChars = 0; maxChars <= text.length; maxChars++) { + const capped = truncateWithoutSplittingSurrogatePair(text, maxChars) + expect(capped.length).toBeLessThanOrEqual(maxChars) + expect(capped).not.toMatch(/[\uD800-\uDBFF]$/) + expect(capped).toBe(text.slice(0, capped.length)) + } + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c5c3824f8c..83a6919e54 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7996,6 +7996,9 @@ importers: '@deepseek-ai/dsh-llm': specifier: workspace:^ version: link:../../llm/llm + '@deepseek-ai/dsh-output-retention': + specifier: workspace:^ + version: link:../../util/output-retention '@deepseek-ai/dsh-sandbox': specifier: workspace:^ version: link:../../sandbox/sandbox @@ -10792,6 +10795,9 @@ importers: '@deepseek-ai/dsh-llm': specifier: workspace:^ version: link:../../llm/llm + '@deepseek-ai/dsh-output-retention': + specifier: workspace:^ + version: link:../../util/output-retention '@deepseek-ai/dsh-sandbox': specifier: workspace:^ version: link:../../sandbox/sandbox @@ -10914,6 +10920,9 @@ importers: '@deepseek-ai/dsh-llm': specifier: workspace:^ version: link:../../llm/llm + '@deepseek-ai/dsh-output-retention': + specifier: workspace:^ + version: link:../../util/output-retention '@deepseek-ai/dsh-pwsh-local': specifier: workspace:^ version: link:../pwsh-local