fix(tools): keep surrogate pairs intact when capping tool output

Three tools capped the preview text they return with String.prototype.slice
at a UTF-16 code-unit budget. A cut that lands inside a surrogate pair
leaves a lone high surrogate, which is not well-formed text: a strict JSON
reader rejects the durable Session log carrying it, and the preview shows a
broken glyph. A lone surrogate retained in history has already blocked every
later Messages request for one session.

Centralize the cap in output-retention as
truncateWithoutSplittingSurrogatePair, and use it from the persistent bash
and pwsh shells and the string-replace editor. The cut drops the unpaired
half instead of keeping it, so the retained text stays a prefix and is at
most one code unit shorter than before.

The new dependency edges and the shifted Config sources regenerate
docs/module-graph and docs/config-catalog, including the committed
translation pair.

Refs #4817
This commit is contained in:
mektpoy committed 2026-09-21 18:51:19 +08:00
1 parent d0ddf939f0
commit dc07e5a50d
24 files changed
+151 -27

No files matched your search

+2 -2
View File
@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write docs/config-catalog.md
config-catalog.md: c46008dd88e51dc079d72e6605d06176c26bdea1
config-catalog.zh.md: 7376476771902e1bbcba3ae4d6374b1087e14a72
config-catalog.md: 6aa27aec967066deb217d1a065b093ba4f09f20f
config-catalog.zh.md: 7051e686c6081c5da65745335bbda2ddef81edce
+3 -3
View File
@@ -3249,7 +3249,7 @@ export interface Config {
}
```
Source: [`packages/shell/tool-bash-persistent/src/index.ts:443`](../packages/shell/tool-bash-persistent/src/index.ts)
Source: [`packages/shell/tool-bash-persistent/src/index.ts:444`](../packages/shell/tool-bash-persistent/src/index.ts)
<a id="deepseek-aidsh-tool-fs"></a>
@@ -3443,7 +3443,7 @@ export interface Config {
}
```
Source: [`packages/shell/tool-pwsh-persistent/src/index.ts:480`](../packages/shell/tool-pwsh-persistent/src/index.ts)
Source: [`packages/shell/tool-pwsh-persistent/src/index.ts:481`](../packages/shell/tool-pwsh-persistent/src/index.ts)
<a id="deepseek-aidsh-tool-ralph"></a>
@@ -3517,7 +3517,7 @@ export interface Config {
}
```
Source: [`packages/fs/tool-str-replace-editor/src/index.ts:505`](../packages/fs/tool-str-replace-editor/src/index.ts)
Source: [`packages/fs/tool-str-replace-editor/src/index.ts:506`](../packages/fs/tool-str-replace-editor/src/index.ts)
<a id="deepseek-aidsh-tool-subagent"></a>
+3 -3
View File
@@ -3251,7 +3251,7 @@ export interface Config {
}
```
来源: [`packages/shell/tool-bash-persistent/src/index.ts:443`](../packages/shell/tool-bash-persistent/src/index.ts)
来源: [`packages/shell/tool-bash-persistent/src/index.ts:444`](../packages/shell/tool-bash-persistent/src/index.ts)
<a id="deepseek-aidsh-tool-fs"></a>
@@ -3445,7 +3445,7 @@ export interface Config {
}
```
来源: [`packages/shell/tool-pwsh-persistent/src/index.ts:480`](../packages/shell/tool-pwsh-persistent/src/index.ts)
来源: [`packages/shell/tool-pwsh-persistent/src/index.ts:481`](../packages/shell/tool-pwsh-persistent/src/index.ts)
<a id="deepseek-aidsh-tool-ralph"></a>
@@ -3519,7 +3519,7 @@ export interface Config {
}
```
来源: [`packages/fs/tool-str-replace-editor/src/index.ts:505`](../packages/fs/tool-str-replace-editor/src/index.ts)
来源: [`packages/fs/tool-str-replace-editor/src/index.ts:506`](../packages/fs/tool-str-replace-editor/src/index.ts)
<a id="deepseek-aidsh-tool-subagent"></a>
+2 -2
View File
@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write docs/module-graph.md
module-graph.md: e36487734f39545fee9b188d9443336ad7ed2e0e
module-graph.zh.md: 646aa21ea348ddf5e73d12b94247319cfe3768d6
module-graph.md: 44e499d26af5753a260d6bc0802ef67adb6de475
module-graph.zh.md: 8e95ddc3a53956c92194c3274484695dcf599619
+6 -3
View File
@@ -780,6 +780,7 @@ flowchart TD
pkg_tool_fs_search --> pkg_timeout
pkg_tool_fs_search --> pkg_tools
pkg_tool_str_replace_editor --> pkg_fs
pkg_tool_str_replace_editor --> pkg_output_retention
pkg_tool_str_replace_editor --> pkg_sandbox
pkg_tool_str_replace_editor --> pkg_sandbox_policy
pkg_tool_str_replace_editor --> pkg_tools
@@ -929,10 +930,12 @@ flowchart TD
pkg_shell_env --> pkg_shell
pkg_shell_env --> pkg_tools
pkg_tool_bash_persistent --> pkg_agent
pkg_tool_bash_persistent --> pkg_output_retention
pkg_tool_bash_persistent --> pkg_terminal
pkg_tool_bash_persistent --> pkg_timeout
pkg_tool_bash_persistent --> pkg_tools
pkg_tool_pwsh_persistent --> pkg_agent
pkg_tool_pwsh_persistent --> pkg_output_retention
pkg_tool_pwsh_persistent --> pkg_terminal
pkg_tool_pwsh_persistent --> pkg_timeout
pkg_tool_pwsh_persistent --> pkg_tools
@@ -1540,7 +1543,7 @@ flowchart TD
| [`tool-goal`](../packages/goal/tool-goal) | `goal` | [`agent`](../packages/core/agent), [`goal`](../packages/goal/goal), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) |
| [`tool-fs`](../packages/fs/tool-fs) | `fs` | [`attachment`](../packages/attachment/attachment), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) |
| [`tool-fs-search`](../packages/fs/tool-fs-search) | `fs` | [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`session`](../packages/core/session), [`spill`](../packages/spill/spill), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) |
| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`output-retention`](../packages/util/output-retention), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) |
| [`tool-skill`](../packages/skill/tool-skill) | `skill` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`skill`](../packages/skill/skill), [`tools`](../packages/core/tools) |
| [`tool-workspace-dependencies`](../packages/skill/tool-workspace-dependencies) | `skill` | [`tools`](../packages/core/tools) |
| [`tool-web`](../packages/web/tool-web) | `web` | [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`web`](../packages/web/web) |
@@ -1572,8 +1575,8 @@ flowchart TD
| [`session-title-all-prompts-llm`](../packages/session/session-title-all-prompts-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) |
| [`session-title-first-prompt-llm`](../packages/session/session-title-first-prompt-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) |
| [`shell-env`](../packages/shell/shell-env) | `shell` | [`home-paths`](../packages/util/home-paths), [`shell`](../packages/shell/shell), [`tools`](../packages/core/tools) |
| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`ssh`](../packages/ssh/ssh) | `ssh` | [`brand`](../packages/util/brand), [`fs`](../packages/fs/fs), [`fs-local`](../packages/fs/fs-local), [`fs-sandbox`](../packages/fs/fs-sandbox), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-local`](../packages/sandbox/sandbox-local), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session-projection`](../packages/session/session-projection), [`subprocess`](../packages/subprocess/subprocess), [`subprocess-local`](../packages/subprocess/subprocess-local) |
| [`tool-terminal`](../packages/terminal/tool-terminal) | `terminal` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`system-prompt`](../packages/core/system-prompt), [`terminal`](../packages/terminal/terminal), [`tools`](../packages/core/tools) |
| [`llm-replay`](../packages/test-support/llm-replay) | `test-support` | [`compaction`](../packages/compaction/compaction), [`deepseek-llm-api-extensions`](../packages/llm/deepseek-llm-api-extensions), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) |
+6 -3
View File
@@ -782,6 +782,7 @@ flowchart TD
pkg_tool_fs_search --> pkg_timeout
pkg_tool_fs_search --> pkg_tools
pkg_tool_str_replace_editor --> pkg_fs
pkg_tool_str_replace_editor --> pkg_output_retention
pkg_tool_str_replace_editor --> pkg_sandbox
pkg_tool_str_replace_editor --> pkg_sandbox_policy
pkg_tool_str_replace_editor --> pkg_tools
@@ -931,10 +932,12 @@ flowchart TD
pkg_shell_env --> pkg_shell
pkg_shell_env --> pkg_tools
pkg_tool_bash_persistent --> pkg_agent
pkg_tool_bash_persistent --> pkg_output_retention
pkg_tool_bash_persistent --> pkg_terminal
pkg_tool_bash_persistent --> pkg_timeout
pkg_tool_bash_persistent --> pkg_tools
pkg_tool_pwsh_persistent --> pkg_agent
pkg_tool_pwsh_persistent --> pkg_output_retention
pkg_tool_pwsh_persistent --> pkg_terminal
pkg_tool_pwsh_persistent --> pkg_timeout
pkg_tool_pwsh_persistent --> pkg_tools
@@ -1542,7 +1545,7 @@ flowchart TD
| [`tool-goal`](../packages/goal/tool-goal) | `goal` | [`agent`](../packages/core/agent), [`goal`](../packages/goal/goal), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) |
| [`tool-fs`](../packages/fs/tool-fs) | `fs` | [`attachment`](../packages/attachment/attachment), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) |
| [`tool-fs-search`](../packages/fs/tool-fs-search) | `fs` | [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`session`](../packages/core/session), [`spill`](../packages/spill/spill), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) |
| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`output-retention`](../packages/util/output-retention), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) |
| [`tool-skill`](../packages/skill/tool-skill) | `skill` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`skill`](../packages/skill/skill), [`tools`](../packages/core/tools) |
| [`tool-workspace-dependencies`](../packages/skill/tool-workspace-dependencies) | `skill` | [`tools`](../packages/core/tools) |
| [`tool-web`](../packages/web/tool-web) | `web` | [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`web`](../packages/web/web) |
@@ -1574,8 +1577,8 @@ flowchart TD
| [`session-title-all-prompts-llm`](../packages/session/session-title-all-prompts-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) |
| [`session-title-first-prompt-llm`](../packages/session/session-title-first-prompt-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) |
| [`shell-env`](../packages/shell/shell-env) | `shell` | [`home-paths`](../packages/util/home-paths), [`shell`](../packages/shell/shell), [`tools`](../packages/core/tools) |
| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
| [`ssh`](../packages/ssh/ssh) | `ssh` | [`brand`](../packages/util/brand), [`fs`](../packages/fs/fs), [`fs-local`](../packages/fs/fs-local), [`fs-sandbox`](../packages/fs/fs-sandbox), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-local`](../packages/sandbox/sandbox-local), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session-projection`](../packages/session/session-projection), [`subprocess`](../packages/subprocess/subprocess), [`subprocess-local`](../packages/subprocess/subprocess-local) |
| [`tool-terminal`](../packages/terminal/tool-terminal) | `terminal` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`system-prompt`](../packages/core/system-prompt), [`terminal`](../packages/terminal/terminal), [`tools`](../packages/core/tools) |
| [`llm-replay`](../packages/test-support/llm-replay) | `test-support` | [`compaction`](../packages/compaction/compaction), [`deepseek-llm-api-extensions`](../packages/llm/deepseek-llm-api-extensions), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) |
@@ -27,6 +27,7 @@
"license": "MIT",
"peerDependencies": {
"@deepseek-ai/dsh-fs": "workspace:^",
"@deepseek-ai/dsh-output-retention": "workspace:^",
"@deepseek-ai/dsh-sandbox": "workspace:^",
"@deepseek-ai/dsh-sandbox-policy": "workspace:^",
"@deepseek-ai/dsh-tools": "workspace:^",
@@ -43,6 +44,7 @@
"@deepseek-ai/dsh-fs-observation-policy": "workspace:^",
"@deepseek-ai/dsh-fs-sandbox": "workspace:^",
"@deepseek-ai/dsh-llm": "workspace:^",
"@deepseek-ai/dsh-output-retention": "workspace:^",
"@deepseek-ai/dsh-sandbox": "workspace:^",
"@deepseek-ai/dsh-sandbox-policy": "workspace:^",
"@deepseek-ai/dsh-session": "workspace:^",
@@ -8,6 +8,7 @@ import type { Context } from '@deepseek-ai/cordis'
import z from '@deepseek-ai/schemastery'
import { FsError } from '@deepseek-ai/dsh-fs'
import type { FsInfo, FsTarget, FsWriteIntent } from '@deepseek-ai/dsh-fs'
import { truncateWithoutSplittingSurrogatePair } from '@deepseek-ai/dsh-output-retention'
import { sandboxDenialMarker } from '@deepseek-ai/dsh-sandbox'
import type { SandboxExecutionPolicy } from '@deepseek-ai/dsh-sandbox'
import type { SandboxPolicyService } from '@deepseek-ai/dsh-sandbox-policy'
@@ -33,7 +34,7 @@ Notes for using the \`str_replace\` command:
function maybeTruncate(content: string, maxOutputChars: number): string {
return content.length <= maxOutputChars
? content
: content.slice(0, maxOutputChars) + TRUNCATED_MESSAGE
: truncateWithoutSplittingSurrogatePair(content, maxOutputChars) + TRUNCATED_MESSAGE
}
function codepointCompare(left: string, right: string): number {
@@ -347,6 +347,21 @@ describe('tool-str-replace-editor', () => {
.toContain('<response clipped>')
})
it('clips a view at a code-unit boundary without splitting a surrogate pair', async () => {
const wide = await setup({ maxOutputChars: 10_000 })
const file = join(wide.root, 'emoji.txt')
await writeFile(file, `${'😀'.repeat(4)}tail`)
const untruncated = text(await call(wide.ctx, wide.owner, { command: 'view', path: file }))
// The cap lands on the high surrogate of the first emoji.
const splitAt = untruncated.indexOf('😀') + 1
const clipped = await setup({ maxOutputChars: splitAt })
const rendered = text(await call(clipped.ctx, clipped.owner, { command: 'view', path: file }))
expect(rendered.startsWith(`${untruncated.slice(0, splitAt - 1)}<response clipped>`)).toBe(true)
expect(rendered).not.toContain('\uD83D')
})
it('matches canonical empty-line, range, and end-insert behavior', async () => {
const { ctx, root, owner } = await setup()
const empty = join(root, 'empty.txt')
@@ -8,6 +8,7 @@
"references": [
{ "path": "../../../vendor/cordis" },
{ "path": "../../../vendor/schemastery" },
{ "path": "../../util/output-retention" },
{ "path": "../../core/tools" },
{ "path": "../fs" },
{ "path": "../../sandbox/sandbox" },
@@ -27,6 +27,7 @@
"license": "MIT",
"peerDependencies": {
"@deepseek-ai/dsh-agent": "workspace:^",
"@deepseek-ai/dsh-output-retention": "workspace:^",
"@deepseek-ai/dsh-terminal": "workspace:^",
"@deepseek-ai/dsh-timeout": "workspace:^",
"@deepseek-ai/dsh-tools": "workspace:^",
@@ -41,6 +42,7 @@
"@deepseek-ai/dsh-agent": "workspace:^",
"@deepseek-ai/dsh-agent-loop-testkit": "workspace:^",
"@deepseek-ai/dsh-llm": "workspace:^",
"@deepseek-ai/dsh-output-retention": "workspace:^",
"@deepseek-ai/dsh-terminal": "workspace:^",
"@deepseek-ai/dsh-terminal-bash": "workspace:^",
"@deepseek-ai/dsh-sandbox": "workspace:^",
@@ -7,6 +7,7 @@ import { randomUUID } from 'node:crypto'
import type { Context } from '@deepseek-ai/cordis'
import z from '@deepseek-ai/schemastery'
import type { Agent } from '@deepseek-ai/dsh-agent'
import { truncateWithoutSplittingSurrogatePair } from '@deepseek-ai/dsh-output-retention'
import type { TerminalReadResult, TerminalSessionId } from '@deepseek-ai/dsh-terminal'
import { deadline, timeoutOf } from '@deepseek-ai/dsh-timeout'
import { defineTool } from '@deepseek-ai/dsh-tools'
@@ -58,7 +59,7 @@ function maybeTruncate(content: string, maxOutputChars: number, incomplete = fal
if (content.length <= maxOutputChars && !incomplete) return content
return content.length <= maxOutputChars
? content + TRUNCATED_MESSAGE
: content.slice(0, maxOutputChars) + TRUNCATED_MESSAGE
: truncateWithoutSplittingSurrogatePair(content, maxOutputChars) + TRUNCATED_MESSAGE
}
function markers(): CommandMarkers {
@@ -114,6 +114,7 @@ class StubPtySession implements TerminalBackendSession {
pendingText = ''
historyTruncated = false
throwOnSend = false
largeOutput = 'x'.repeat(100)
constructor(mode: StubMode) {
this.mode = mode
@@ -206,7 +207,7 @@ class StubPtySession implements TerminalBackendSession {
return this.operation(Promise.resolve(this.result(output, 'stdin_read')))
}
const commandOutput = this.mode === 'large'
? 'x'.repeat(100)
? this.largeOutput
: this.mode === 'nonzero' ? '' : 'hello from stub'
const exitCode = this.mode === 'nonzero' ? 7 : 0
const output = `${start ?? ''}\n${commandOutput}\n${end ?? ''}${exitCode}\n${this.motd}`
@@ -404,6 +405,21 @@ describe('tool-bash-persistent', () => {
expect(stub.sessions[2]?.closed).toEqual(['external cleanup'])
})
it('clips a split surrogate pair instead of leaving a lone half', async () => {
const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 10 })
await call(ctx, owner, 'warm up')
const session = stub.sessions[0]!
session.mode = 'large'
// The tenth code unit is the emoji's high surrogate, so the clip drops the
// unpaired half instead of emitting it.
session.largeOutput = `${'x'.repeat(9)}😀tail`
const rendered = text(await call(ctx, owner, 'emoji'))
expect(rendered.startsWith(`${'x'.repeat(9)}<response clipped>`)).toBe(true)
expect(rendered).not.toContain('\uD83D')
})
it('waits for status digits after a torn completion marker', async () => {
const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 1_000 })
await call(ctx, owner, 'warm up')
@@ -8,6 +8,7 @@
"references": [
{ "path": "../../../vendor/cordis" },
{ "path": "../../../vendor/schemastery" },
{ "path": "../../util/output-retention" },
{ "path": "../../core/agent" },
{ "path": "../../core/tools" },
{ "path": "../../terminal/terminal" },
@@ -27,6 +27,7 @@
"license": "MIT",
"peerDependencies": {
"@deepseek-ai/dsh-agent": "workspace:^",
"@deepseek-ai/dsh-output-retention": "workspace:^",
"@deepseek-ai/dsh-terminal": "workspace:^",
"@deepseek-ai/dsh-timeout": "workspace:^",
"@deepseek-ai/dsh-tools": "workspace:^",
@@ -41,6 +42,7 @@
"@deepseek-ai/dsh-agent": "workspace:^",
"@deepseek-ai/dsh-agent-loop-testkit": "workspace:^",
"@deepseek-ai/dsh-llm": "workspace:^",
"@deepseek-ai/dsh-output-retention": "workspace:^",
"@deepseek-ai/dsh-terminal": "workspace:^",
"@deepseek-ai/dsh-terminal-bash": "workspace:^",
"@deepseek-ai/dsh-pwsh-local": "workspace:^",
@@ -9,6 +9,7 @@ import { randomUUID } from 'node:crypto'
import type { Context } from '@deepseek-ai/cordis'
import z from '@deepseek-ai/schemastery'
import type { Agent } from '@deepseek-ai/dsh-agent'
import { truncateWithoutSplittingSurrogatePair } from '@deepseek-ai/dsh-output-retention'
import type { TerminalReadResult, TerminalSendResult, TerminalSessionId } from '@deepseek-ai/dsh-terminal'
import { deadline, timeoutOf } from '@deepseek-ai/dsh-timeout'
import { defineTool } from '@deepseek-ai/dsh-tools'
@@ -58,7 +59,7 @@ function maybeTruncate(content: string, maxOutputChars: number, incomplete = fal
if (content.length <= maxOutputChars && !incomplete) return content
return content.length <= maxOutputChars
? content + TRUNCATED_MESSAGE
: content.slice(0, maxOutputChars) + TRUNCATED_MESSAGE
: truncateWithoutSplittingSurrogatePair(content, maxOutputChars) + TRUNCATED_MESSAGE
}
function markers(): CommandMarkers {
@@ -119,6 +119,7 @@ class StubTerminalSession implements TerminalBackendSession {
pendingText = ''
historyTruncated = false
throwOnSend = false
largeOutput = 'x'.repeat(100)
constructor(mode: StubMode) {
this.mode = mode
@@ -218,7 +219,7 @@ class StubTerminalSession implements TerminalBackendSession {
return this.operation(Promise.resolve(this.result(output, 'stdin_read')))
}
const commandOutput = this.mode === 'large'
? 'x'.repeat(100)
? this.largeOutput
: this.mode === 'nonzero' ? ''
: this.mode === 'prompt-collision' ? this.motd
: 'hello from stub'
@@ -455,6 +456,21 @@ describe('tool-pwsh-persistent', () => {
expect(stub.sessions[2]?.closed).toEqual(['external cleanup'])
})
it('clips a split surrogate pair instead of leaving a lone half', async () => {
const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 10 })
await call(ctx, owner, 'warm up')
const session = stub.sessions[0]!
session.mode = 'large'
// The tenth code unit is the emoji's high surrogate, so the clip drops the
// unpaired half instead of emitting it.
session.largeOutput = `${'x'.repeat(9)}😀tail`
const rendered = text(await call(ctx, owner, 'emoji'))
expect(rendered.startsWith(`${'x'.repeat(9)}<response clipped>`)).toBe(true)
expect(rendered).not.toContain('\uD83D')
})
it('waits for status digits after a torn completion marker', async () => {
const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 1_000 })
await call(ctx, owner, 'warm up')
@@ -8,6 +8,7 @@
"references": [
{ "path": "../../../vendor/cordis" },
{ "path": "../../../vendor/schemastery" },
{ "path": "../../util/output-retention" },
{ "path": "../../core/agent" },
{ "path": "../../core/tools" },
{ "path": "../../terminal/terminal" },
@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write packages/util/output-retention/README.md
README.md: bb0902b959f298f9f7b0a64de23d48e29bfd75ad
README.zh.md: 7d96ba4da33a98b24eba53f9dd5a2f2c98b79f39
README.md: ec16dd9626f2268a9a019373c3e135619e1ef2e5
README.zh.md: 1ae37399224838d6b044dacc81a15983f82ff9c8
+2 -2
View File
@@ -9,7 +9,7 @@ English | [中文](README.zh.md)
## Summary
Use `dsh-output-retention` to cap the items or text a tool returns to a model while reporting what was omitted. `ItemRetainer` keeps an ordered head window and can report an exact omitted-item count; `TextRetainer` keeps head, tail, or head-and-tail byte windows without returning invalid UTF-8 cuts. `formatRetentionNotice` adds a consistent omission clause while each tool supplies its own recovery guidance. Grouping, line numbering, spill files, and provider errors remain tool responsibilities; consumers import this library directly rather than loading it through `cordis.yml`.
Use `dsh-output-retention` to cap the items or text a tool returns to a model while reporting what was omitted. `ItemRetainer` keeps an ordered head window and can report an exact omitted-item count; `TextRetainer` keeps head, tail, or head-and-tail byte windows without returning invalid UTF-8 cuts. `formatRetentionNotice` adds a consistent omission clause while each tool supplies its own recovery guidance. `truncateWithoutSplittingSurrogatePair` caps a character-budget preview without leaving a lone high surrogate at the cut. Grouping, line numbering, spill files, and provider errors remain tool responsibilities; consumers import this library directly rather than loading it through `cordis.yml`.
## Table of Contents
@@ -104,7 +104,7 @@ The library is built on one separation: it owns the mechanical question of what
| File | Role |
|---|---|
| [`src/index.ts`](src/index.ts) | `ItemRetainer`, `TextRetainer`, `describeOmitted`, and `formatRetentionNotice` |
| [`src/index.ts`](src/index.ts) | `ItemRetainer`, `TextRetainer`, `describeOmitted`, `formatRetentionNotice`, and `truncateWithoutSplittingSurrogatePair` |
| — | No runtime invariant companion is published; this pure utility owns no event stream or mutable runtime data; its value algebra is enforced by unit tests. |
### Two retainers, two resource models
+2 -2
View File
@@ -9,7 +9,7 @@ kind: "package-library"
## 概述
使用 `dsh-output-retention` 限制工具返回给模型的项或文本量,并报告省略了什么。`ItemRetainer` 保留有序的头部窗口,并可报告精确的省略项数;`TextRetainer` 保留 head、tail 或 head-and-tail 字节窗口,且不会返回因切割而无效的 UTF-8。`formatRetentionNotice` 添加一致的省略子句,各工具则提供自己的恢复指引。分组、行号、spill 文件与提供方错误仍归工具负责;消费方直接导入本库,而不通过 `cordis.yml` 加载。
使用 `dsh-output-retention` 限制工具返回给模型的项或文本量,并报告省略了什么。`ItemRetainer` 保留有序的头部窗口,并可报告精确的省略项数;`TextRetainer` 保留 head、tail 或 head-and-tail 字节窗口,且不会返回因切割而无效的 UTF-8。`formatRetentionNotice` 添加一致的省略子句,各工具则提供自己的恢复指引。`truncateWithoutSplittingSurrogatePair` 按字符预算截断预览时,不会在切口留下孤立的高位代理项。分组、行号、spill 文件与提供方错误仍归工具负责;消费方直接导入本库,而不通过 `cordis.yml` 加载。
## 目录
@@ -104,7 +104,7 @@ const footer = formatRetentionNotice(
| 文件 | 职责 |
|---|---|
| [`src/index.ts`](src/index.ts) | `ItemRetainer`、`TextRetainer`、`describeOmitted` 与 `formatRetentionNotice` |
| [`src/index.ts`](src/index.ts) | `ItemRetainer`、`TextRetainer`、`describeOmitted`、`formatRetentionNotice` 与 `truncateWithoutSplittingSurrogatePair` |
| — | 不发布运行时不变式伴生入口;这个纯工具不拥有事件流或可变运行时数据;其值代数由单元测试保证。 |
### 两个 retainer,两种资源模型
@@ -441,3 +441,20 @@ export function formatRetentionNotice(
.filter(part => part.length > 0)
.join(' ')
}
/**
* Cap `text` at `maxChars` UTF-16 code units without ending in a lone high
* surrogate: a cut that lands inside a surrogate pair drops the unpaired half,
* so the result is one code unit shorter than the cap. An unpaired surrogate is
* not well-formed text — a strict JSON reader rejects a durable Session log that
* carries one.
*
* @param text The text to cap.
* @param maxChars The maximum number of UTF-16 code units to retain.
* @returns A prefix of `text` no longer than `maxChars`, never ending in a lone high surrogate.
*/
export function truncateWithoutSplittingSurrogatePair(text: string, maxChars: number): string {
if (text.length <= maxChars) return text
const capped = text.slice(0, maxChars)
return /[\uD800-\uDBFF]$/.test(capped) ? capped.slice(0, -1) : capped
}
@@ -6,6 +6,7 @@ import {
type Omitted,
type RetentionNotice,
TextRetainer,
truncateWithoutSplittingSurrogatePair,
} from '@deepseek-ai/dsh-output-retention'
/** Decode a RetainedText via a round-trip helper for readable UTF-8 assertions. */
@@ -374,3 +375,35 @@ describe('formatRetentionNotice', () => {
expect(out).toBe('Omitted 500 bytes. Kept 2000B head + 2000B tail.')
})
})
describe('truncateWithoutSplittingSurrogatePair', () => {
it('caps at maxChars UTF-16 code units', () => {
expect(truncateWithoutSplittingSurrogatePair('abcdef', 3)).toBe('abc')
expect(truncateWithoutSplittingSurrogatePair('abcdef', 0)).toBe('')
expect(truncateWithoutSplittingSurrogatePair('abcdef', 1)).toBe('a')
})
it('returns text within the cap unchanged', () => {
expect(truncateWithoutSplittingSurrogatePair('abc', 3)).toBe('abc')
expect(truncateWithoutSplittingSurrogatePair('abc', 9)).toBe('abc')
expect(truncateWithoutSplittingSurrogatePair('', 0)).toBe('')
})
it('drops the unpaired half when the cap lands inside a surrogate pair', () => {
// '😀' is two code units, so these caps would otherwise keep its high surrogate alone.
expect(truncateWithoutSplittingSurrogatePair('ab😀cd', 3)).toBe('ab')
expect(truncateWithoutSplittingSurrogatePair('ab😀cd', 4)).toBe('ab😀')
expect(truncateWithoutSplittingSurrogatePair('😀😀', 1)).toBe('')
expect(truncateWithoutSplittingSurrogatePair('😀😀', 3)).toBe('😀')
})
it('never ends in a lone high surrogate at any cap', () => {
const text = 'a😀b😀c'
for (let maxChars = 0; maxChars <= text.length; maxChars++) {
const capped = truncateWithoutSplittingSurrogatePair(text, maxChars)
expect(capped.length).toBeLessThanOrEqual(maxChars)
expect(capped).not.toMatch(/[\uD800-\uDBFF]$/)
expect(capped).toBe(text.slice(0, capped.length))
}
})
})
+9
View File
@@ -7996,6 +7996,9 @@ importers:
'@deepseek-ai/dsh-llm':
specifier: workspace:^
version: link:../../llm/llm
'@deepseek-ai/dsh-output-retention':
specifier: workspace:^
version: link:../../util/output-retention
'@deepseek-ai/dsh-sandbox':
specifier: workspace:^
version: link:../../sandbox/sandbox
@@ -10792,6 +10795,9 @@ importers:
'@deepseek-ai/dsh-llm':
specifier: workspace:^
version: link:../../llm/llm
'@deepseek-ai/dsh-output-retention':
specifier: workspace:^
version: link:../../util/output-retention
'@deepseek-ai/dsh-sandbox':
specifier: workspace:^
version: link:../../sandbox/sandbox
@@ -10914,6 +10920,9 @@ importers:
'@deepseek-ai/dsh-llm':
specifier: workspace:^
version: link:../../llm/llm
'@deepseek-ai/dsh-output-retention':
specifier: workspace:^
version: link:../../util/output-retention
'@deepseek-ai/dsh-pwsh-local':
specifier: workspace:^
version: link:../pwsh-local