fix(tools): keep surrogate pairs intact when capping tool output
Three tools capped the preview text they return with String.prototype.slice at a UTF-16 code-unit budget. A cut that lands inside a surrogate pair leaves a lone high surrogate, which is not well-formed text: a strict JSON reader rejects the durable Session log carrying it, and the preview shows a broken glyph. A lone surrogate retained in history has already blocked every later Messages request for one session. Centralize the cap in output-retention as truncateWithoutSplittingSurrogatePair, and use it from the persistent bash and pwsh shells and the string-replace editor. The cut drops the unpaired half instead of keeping it, so the retained text stays a prefix and is at most one code unit shorter than before. The new dependency edges and the shifted Config sources regenerate docs/module-graph and docs/config-catalog, including the committed translation pair. Refs #4817
This commit is contained in:
1 parent
d0ddf939f0
commit
dc07e5a50d
24 files changed
+151
-27
No files matched your search
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write docs/config-catalog.md
|
||||
config-catalog.md: c46008dd88e51dc079d72e6605d06176c26bdea1
|
||||
config-catalog.zh.md: 7376476771902e1bbcba3ae4d6374b1087e14a72
|
||||
config-catalog.md: 6aa27aec967066deb217d1a065b093ba4f09f20f
|
||||
config-catalog.zh.md: 7051e686c6081c5da65745335bbda2ddef81edce
|
||||
@@ -3249,7 +3249,7 @@ export interface Config {
|
||||
}
|
||||
```
|
||||
|
||||
Source: [`packages/shell/tool-bash-persistent/src/index.ts:443`](../packages/shell/tool-bash-persistent/src/index.ts)
|
||||
Source: [`packages/shell/tool-bash-persistent/src/index.ts:444`](../packages/shell/tool-bash-persistent/src/index.ts)
|
||||
|
||||
<a id="deepseek-aidsh-tool-fs"></a>
|
||||
|
||||
@@ -3443,7 +3443,7 @@ export interface Config {
|
||||
}
|
||||
```
|
||||
|
||||
Source: [`packages/shell/tool-pwsh-persistent/src/index.ts:480`](../packages/shell/tool-pwsh-persistent/src/index.ts)
|
||||
Source: [`packages/shell/tool-pwsh-persistent/src/index.ts:481`](../packages/shell/tool-pwsh-persistent/src/index.ts)
|
||||
|
||||
<a id="deepseek-aidsh-tool-ralph"></a>
|
||||
|
||||
@@ -3517,7 +3517,7 @@ export interface Config {
|
||||
}
|
||||
```
|
||||
|
||||
Source: [`packages/fs/tool-str-replace-editor/src/index.ts:505`](../packages/fs/tool-str-replace-editor/src/index.ts)
|
||||
Source: [`packages/fs/tool-str-replace-editor/src/index.ts:506`](../packages/fs/tool-str-replace-editor/src/index.ts)
|
||||
|
||||
<a id="deepseek-aidsh-tool-subagent"></a>
|
||||
|
||||
|
||||
@@ -3251,7 +3251,7 @@ export interface Config {
|
||||
}
|
||||
```
|
||||
|
||||
来源: [`packages/shell/tool-bash-persistent/src/index.ts:443`](../packages/shell/tool-bash-persistent/src/index.ts)
|
||||
来源: [`packages/shell/tool-bash-persistent/src/index.ts:444`](../packages/shell/tool-bash-persistent/src/index.ts)
|
||||
|
||||
<a id="deepseek-aidsh-tool-fs"></a>
|
||||
|
||||
@@ -3445,7 +3445,7 @@ export interface Config {
|
||||
}
|
||||
```
|
||||
|
||||
来源: [`packages/shell/tool-pwsh-persistent/src/index.ts:480`](../packages/shell/tool-pwsh-persistent/src/index.ts)
|
||||
来源: [`packages/shell/tool-pwsh-persistent/src/index.ts:481`](../packages/shell/tool-pwsh-persistent/src/index.ts)
|
||||
|
||||
<a id="deepseek-aidsh-tool-ralph"></a>
|
||||
|
||||
@@ -3519,7 +3519,7 @@ export interface Config {
|
||||
}
|
||||
```
|
||||
|
||||
来源: [`packages/fs/tool-str-replace-editor/src/index.ts:505`](../packages/fs/tool-str-replace-editor/src/index.ts)
|
||||
来源: [`packages/fs/tool-str-replace-editor/src/index.ts:506`](../packages/fs/tool-str-replace-editor/src/index.ts)
|
||||
|
||||
<a id="deepseek-aidsh-tool-subagent"></a>
|
||||
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write docs/module-graph.md
|
||||
module-graph.md: e36487734f39545fee9b188d9443336ad7ed2e0e
|
||||
module-graph.zh.md: 646aa21ea348ddf5e73d12b94247319cfe3768d6
|
||||
module-graph.md: 44e499d26af5753a260d6bc0802ef67adb6de475
|
||||
module-graph.zh.md: 8e95ddc3a53956c92194c3274484695dcf599619
|
||||
@@ -780,6 +780,7 @@ flowchart TD
|
||||
pkg_tool_fs_search --> pkg_timeout
|
||||
pkg_tool_fs_search --> pkg_tools
|
||||
pkg_tool_str_replace_editor --> pkg_fs
|
||||
pkg_tool_str_replace_editor --> pkg_output_retention
|
||||
pkg_tool_str_replace_editor --> pkg_sandbox
|
||||
pkg_tool_str_replace_editor --> pkg_sandbox_policy
|
||||
pkg_tool_str_replace_editor --> pkg_tools
|
||||
@@ -929,10 +930,12 @@ flowchart TD
|
||||
pkg_shell_env --> pkg_shell
|
||||
pkg_shell_env --> pkg_tools
|
||||
pkg_tool_bash_persistent --> pkg_agent
|
||||
pkg_tool_bash_persistent --> pkg_output_retention
|
||||
pkg_tool_bash_persistent --> pkg_terminal
|
||||
pkg_tool_bash_persistent --> pkg_timeout
|
||||
pkg_tool_bash_persistent --> pkg_tools
|
||||
pkg_tool_pwsh_persistent --> pkg_agent
|
||||
pkg_tool_pwsh_persistent --> pkg_output_retention
|
||||
pkg_tool_pwsh_persistent --> pkg_terminal
|
||||
pkg_tool_pwsh_persistent --> pkg_timeout
|
||||
pkg_tool_pwsh_persistent --> pkg_tools
|
||||
@@ -1540,7 +1543,7 @@ flowchart TD
|
||||
| [`tool-goal`](../packages/goal/tool-goal) | `goal` | [`agent`](../packages/core/agent), [`goal`](../packages/goal/goal), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) |
|
||||
| [`tool-fs`](../packages/fs/tool-fs) | `fs` | [`attachment`](../packages/attachment/attachment), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) |
|
||||
| [`tool-fs-search`](../packages/fs/tool-fs-search) | `fs` | [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`session`](../packages/core/session), [`spill`](../packages/spill/spill), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) |
|
||||
| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`output-retention`](../packages/util/output-retention), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) |
|
||||
| [`tool-skill`](../packages/skill/tool-skill) | `skill` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`skill`](../packages/skill/skill), [`tools`](../packages/core/tools) |
|
||||
| [`tool-workspace-dependencies`](../packages/skill/tool-workspace-dependencies) | `skill` | [`tools`](../packages/core/tools) |
|
||||
| [`tool-web`](../packages/web/tool-web) | `web` | [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`web`](../packages/web/web) |
|
||||
@@ -1572,8 +1575,8 @@ flowchart TD
|
||||
| [`session-title-all-prompts-llm`](../packages/session/session-title-all-prompts-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) |
|
||||
| [`session-title-first-prompt-llm`](../packages/session/session-title-first-prompt-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) |
|
||||
| [`shell-env`](../packages/shell/shell-env) | `shell` | [`home-paths`](../packages/util/home-paths), [`shell`](../packages/shell/shell), [`tools`](../packages/core/tools) |
|
||||
| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`ssh`](../packages/ssh/ssh) | `ssh` | [`brand`](../packages/util/brand), [`fs`](../packages/fs/fs), [`fs-local`](../packages/fs/fs-local), [`fs-sandbox`](../packages/fs/fs-sandbox), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-local`](../packages/sandbox/sandbox-local), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session-projection`](../packages/session/session-projection), [`subprocess`](../packages/subprocess/subprocess), [`subprocess-local`](../packages/subprocess/subprocess-local) |
|
||||
| [`tool-terminal`](../packages/terminal/tool-terminal) | `terminal` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`system-prompt`](../packages/core/system-prompt), [`terminal`](../packages/terminal/terminal), [`tools`](../packages/core/tools) |
|
||||
| [`llm-replay`](../packages/test-support/llm-replay) | `test-support` | [`compaction`](../packages/compaction/compaction), [`deepseek-llm-api-extensions`](../packages/llm/deepseek-llm-api-extensions), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) |
|
||||
|
||||
@@ -782,6 +782,7 @@ flowchart TD
|
||||
pkg_tool_fs_search --> pkg_timeout
|
||||
pkg_tool_fs_search --> pkg_tools
|
||||
pkg_tool_str_replace_editor --> pkg_fs
|
||||
pkg_tool_str_replace_editor --> pkg_output_retention
|
||||
pkg_tool_str_replace_editor --> pkg_sandbox
|
||||
pkg_tool_str_replace_editor --> pkg_sandbox_policy
|
||||
pkg_tool_str_replace_editor --> pkg_tools
|
||||
@@ -931,10 +932,12 @@ flowchart TD
|
||||
pkg_shell_env --> pkg_shell
|
||||
pkg_shell_env --> pkg_tools
|
||||
pkg_tool_bash_persistent --> pkg_agent
|
||||
pkg_tool_bash_persistent --> pkg_output_retention
|
||||
pkg_tool_bash_persistent --> pkg_terminal
|
||||
pkg_tool_bash_persistent --> pkg_timeout
|
||||
pkg_tool_bash_persistent --> pkg_tools
|
||||
pkg_tool_pwsh_persistent --> pkg_agent
|
||||
pkg_tool_pwsh_persistent --> pkg_output_retention
|
||||
pkg_tool_pwsh_persistent --> pkg_terminal
|
||||
pkg_tool_pwsh_persistent --> pkg_timeout
|
||||
pkg_tool_pwsh_persistent --> pkg_tools
|
||||
@@ -1542,7 +1545,7 @@ flowchart TD
|
||||
| [`tool-goal`](../packages/goal/tool-goal) | `goal` | [`agent`](../packages/core/agent), [`goal`](../packages/goal/goal), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-projection`](../packages/session/session-projection), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) |
|
||||
| [`tool-fs`](../packages/fs/tool-fs) | `fs` | [`attachment`](../packages/attachment/attachment), [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) |
|
||||
| [`tool-fs-search`](../packages/fs/tool-fs-search) | `fs` | [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`session`](../packages/core/session), [`spill`](../packages/spill/spill), [`subprocess`](../packages/subprocess/subprocess), [`system-prompt`](../packages/core/system-prompt), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) |
|
||||
| [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) | `fs` | [`fs`](../packages/fs/fs), [`output-retention`](../packages/util/output-retention), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`tools`](../packages/core/tools) |
|
||||
| [`tool-skill`](../packages/skill/tool-skill) | `skill` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`skill`](../packages/skill/skill), [`tools`](../packages/core/tools) |
|
||||
| [`tool-workspace-dependencies`](../packages/skill/tool-workspace-dependencies) | `skill` | [`tools`](../packages/core/tools) |
|
||||
| [`tool-web`](../packages/web/tool-web) | `web` | [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`web`](../packages/web/web) |
|
||||
@@ -1574,8 +1577,8 @@ flowchart TD
|
||||
| [`session-title-all-prompts-llm`](../packages/session/session-title-all-prompts-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) |
|
||||
| [`session-title-first-prompt-llm`](../packages/session/session-title-first-prompt-llm) | `session` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-title`](../packages/session/session-title), [`session-title-llm`](../packages/session/session-title-llm) |
|
||||
| [`shell-env`](../packages/shell/shell-env) | `shell` | [`home-paths`](../packages/util/home-paths), [`shell`](../packages/shell/shell), [`tools`](../packages/core/tools) |
|
||||
| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`tool-bash-persistent`](../packages/shell/tool-bash-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`tool-pwsh-persistent`](../packages/shell/tool-pwsh-persistent) | `shell` | [`agent`](../packages/core/agent), [`output-retention`](../packages/util/output-retention), [`terminal`](../packages/terminal/terminal), [`timeout`](../packages/util/timeout), [`tools`](../packages/core/tools) |
|
||||
| [`ssh`](../packages/ssh/ssh) | `ssh` | [`brand`](../packages/util/brand), [`fs`](../packages/fs/fs), [`fs-local`](../packages/fs/fs-local), [`fs-sandbox`](../packages/fs/fs-sandbox), [`sandbox`](../packages/sandbox/sandbox), [`sandbox-local`](../packages/sandbox/sandbox-local), [`sandbox-policy`](../packages/sandbox/sandbox-policy), [`session-projection`](../packages/session/session-projection), [`subprocess`](../packages/subprocess/subprocess), [`subprocess-local`](../packages/subprocess/subprocess-local) |
|
||||
| [`tool-terminal`](../packages/terminal/tool-terminal) | `terminal` | [`agent`](../packages/core/agent), [`jobs`](../packages/jobs/jobs), [`llm`](../packages/llm/llm), [`output-retention`](../packages/util/output-retention), [`system-prompt`](../packages/core/system-prompt), [`terminal`](../packages/terminal/terminal), [`tools`](../packages/core/tools) |
|
||||
| [`llm-replay`](../packages/test-support/llm-replay) | `test-support` | [`compaction`](../packages/compaction/compaction), [`deepseek-llm-api-extensions`](../packages/llm/deepseek-llm-api-extensions), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) |
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"@deepseek-ai/dsh-fs": "workspace:^",
|
||||
"@deepseek-ai/dsh-output-retention": "workspace:^",
|
||||
"@deepseek-ai/dsh-sandbox": "workspace:^",
|
||||
"@deepseek-ai/dsh-sandbox-policy": "workspace:^",
|
||||
"@deepseek-ai/dsh-tools": "workspace:^",
|
||||
@@ -43,6 +44,7 @@
|
||||
"@deepseek-ai/dsh-fs-observation-policy": "workspace:^",
|
||||
"@deepseek-ai/dsh-fs-sandbox": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm": "workspace:^",
|
||||
"@deepseek-ai/dsh-output-retention": "workspace:^",
|
||||
"@deepseek-ai/dsh-sandbox": "workspace:^",
|
||||
"@deepseek-ai/dsh-sandbox-policy": "workspace:^",
|
||||
"@deepseek-ai/dsh-session": "workspace:^",
|
||||
|
||||
@@ -8,6 +8,7 @@ import type { Context } from '@deepseek-ai/cordis'
|
||||
import z from '@deepseek-ai/schemastery'
|
||||
import { FsError } from '@deepseek-ai/dsh-fs'
|
||||
import type { FsInfo, FsTarget, FsWriteIntent } from '@deepseek-ai/dsh-fs'
|
||||
import { truncateWithoutSplittingSurrogatePair } from '@deepseek-ai/dsh-output-retention'
|
||||
import { sandboxDenialMarker } from '@deepseek-ai/dsh-sandbox'
|
||||
import type { SandboxExecutionPolicy } from '@deepseek-ai/dsh-sandbox'
|
||||
import type { SandboxPolicyService } from '@deepseek-ai/dsh-sandbox-policy'
|
||||
@@ -33,7 +34,7 @@ Notes for using the \`str_replace\` command:
|
||||
function maybeTruncate(content: string, maxOutputChars: number): string {
|
||||
return content.length <= maxOutputChars
|
||||
? content
|
||||
: content.slice(0, maxOutputChars) + TRUNCATED_MESSAGE
|
||||
: truncateWithoutSplittingSurrogatePair(content, maxOutputChars) + TRUNCATED_MESSAGE
|
||||
}
|
||||
|
||||
function codepointCompare(left: string, right: string): number {
|
||||
|
||||
@@ -347,6 +347,21 @@ describe('tool-str-replace-editor', () => {
|
||||
.toContain('<response clipped>')
|
||||
})
|
||||
|
||||
it('clips a view at a code-unit boundary without splitting a surrogate pair', async () => {
|
||||
const wide = await setup({ maxOutputChars: 10_000 })
|
||||
const file = join(wide.root, 'emoji.txt')
|
||||
await writeFile(file, `${'😀'.repeat(4)}tail`)
|
||||
const untruncated = text(await call(wide.ctx, wide.owner, { command: 'view', path: file }))
|
||||
// The cap lands on the high surrogate of the first emoji.
|
||||
const splitAt = untruncated.indexOf('😀') + 1
|
||||
|
||||
const clipped = await setup({ maxOutputChars: splitAt })
|
||||
const rendered = text(await call(clipped.ctx, clipped.owner, { command: 'view', path: file }))
|
||||
|
||||
expect(rendered.startsWith(`${untruncated.slice(0, splitAt - 1)}<response clipped>`)).toBe(true)
|
||||
expect(rendered).not.toContain('\uD83D')
|
||||
})
|
||||
|
||||
it('matches canonical empty-line, range, and end-insert behavior', async () => {
|
||||
const { ctx, root, owner } = await setup()
|
||||
const empty = join(root, 'empty.txt')
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
"references": [
|
||||
{ "path": "../../../vendor/cordis" },
|
||||
{ "path": "../../../vendor/schemastery" },
|
||||
{ "path": "../../util/output-retention" },
|
||||
{ "path": "../../core/tools" },
|
||||
{ "path": "../fs" },
|
||||
{ "path": "../../sandbox/sandbox" },
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"@deepseek-ai/dsh-agent": "workspace:^",
|
||||
"@deepseek-ai/dsh-output-retention": "workspace:^",
|
||||
"@deepseek-ai/dsh-terminal": "workspace:^",
|
||||
"@deepseek-ai/dsh-timeout": "workspace:^",
|
||||
"@deepseek-ai/dsh-tools": "workspace:^",
|
||||
@@ -41,6 +42,7 @@
|
||||
"@deepseek-ai/dsh-agent": "workspace:^",
|
||||
"@deepseek-ai/dsh-agent-loop-testkit": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm": "workspace:^",
|
||||
"@deepseek-ai/dsh-output-retention": "workspace:^",
|
||||
"@deepseek-ai/dsh-terminal": "workspace:^",
|
||||
"@deepseek-ai/dsh-terminal-bash": "workspace:^",
|
||||
"@deepseek-ai/dsh-sandbox": "workspace:^",
|
||||
|
||||
@@ -7,6 +7,7 @@ import { randomUUID } from 'node:crypto'
|
||||
import type { Context } from '@deepseek-ai/cordis'
|
||||
import z from '@deepseek-ai/schemastery'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
import { truncateWithoutSplittingSurrogatePair } from '@deepseek-ai/dsh-output-retention'
|
||||
import type { TerminalReadResult, TerminalSessionId } from '@deepseek-ai/dsh-terminal'
|
||||
import { deadline, timeoutOf } from '@deepseek-ai/dsh-timeout'
|
||||
import { defineTool } from '@deepseek-ai/dsh-tools'
|
||||
@@ -58,7 +59,7 @@ function maybeTruncate(content: string, maxOutputChars: number, incomplete = fal
|
||||
if (content.length <= maxOutputChars && !incomplete) return content
|
||||
return content.length <= maxOutputChars
|
||||
? content + TRUNCATED_MESSAGE
|
||||
: content.slice(0, maxOutputChars) + TRUNCATED_MESSAGE
|
||||
: truncateWithoutSplittingSurrogatePair(content, maxOutputChars) + TRUNCATED_MESSAGE
|
||||
}
|
||||
|
||||
function markers(): CommandMarkers {
|
||||
|
||||
@@ -114,6 +114,7 @@ class StubPtySession implements TerminalBackendSession {
|
||||
pendingText = ''
|
||||
historyTruncated = false
|
||||
throwOnSend = false
|
||||
largeOutput = 'x'.repeat(100)
|
||||
|
||||
constructor(mode: StubMode) {
|
||||
this.mode = mode
|
||||
@@ -206,7 +207,7 @@ class StubPtySession implements TerminalBackendSession {
|
||||
return this.operation(Promise.resolve(this.result(output, 'stdin_read')))
|
||||
}
|
||||
const commandOutput = this.mode === 'large'
|
||||
? 'x'.repeat(100)
|
||||
? this.largeOutput
|
||||
: this.mode === 'nonzero' ? '' : 'hello from stub'
|
||||
const exitCode = this.mode === 'nonzero' ? 7 : 0
|
||||
const output = `${start ?? ''}\n${commandOutput}\n${end ?? ''}${exitCode}\n${this.motd}`
|
||||
@@ -404,6 +405,21 @@ describe('tool-bash-persistent', () => {
|
||||
expect(stub.sessions[2]?.closed).toEqual(['external cleanup'])
|
||||
})
|
||||
|
||||
it('clips a split surrogate pair instead of leaving a lone half', async () => {
|
||||
const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 10 })
|
||||
await call(ctx, owner, 'warm up')
|
||||
const session = stub.sessions[0]!
|
||||
session.mode = 'large'
|
||||
// The tenth code unit is the emoji's high surrogate, so the clip drops the
|
||||
// unpaired half instead of emitting it.
|
||||
session.largeOutput = `${'x'.repeat(9)}😀tail`
|
||||
|
||||
const rendered = text(await call(ctx, owner, 'emoji'))
|
||||
|
||||
expect(rendered.startsWith(`${'x'.repeat(9)}<response clipped>`)).toBe(true)
|
||||
expect(rendered).not.toContain('\uD83D')
|
||||
})
|
||||
|
||||
it('waits for status digits after a torn completion marker', async () => {
|
||||
const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 1_000 })
|
||||
await call(ctx, owner, 'warm up')
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
"references": [
|
||||
{ "path": "../../../vendor/cordis" },
|
||||
{ "path": "../../../vendor/schemastery" },
|
||||
{ "path": "../../util/output-retention" },
|
||||
{ "path": "../../core/agent" },
|
||||
{ "path": "../../core/tools" },
|
||||
{ "path": "../../terminal/terminal" },
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
"license": "MIT",
|
||||
"peerDependencies": {
|
||||
"@deepseek-ai/dsh-agent": "workspace:^",
|
||||
"@deepseek-ai/dsh-output-retention": "workspace:^",
|
||||
"@deepseek-ai/dsh-terminal": "workspace:^",
|
||||
"@deepseek-ai/dsh-timeout": "workspace:^",
|
||||
"@deepseek-ai/dsh-tools": "workspace:^",
|
||||
@@ -41,6 +42,7 @@
|
||||
"@deepseek-ai/dsh-agent": "workspace:^",
|
||||
"@deepseek-ai/dsh-agent-loop-testkit": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm": "workspace:^",
|
||||
"@deepseek-ai/dsh-output-retention": "workspace:^",
|
||||
"@deepseek-ai/dsh-terminal": "workspace:^",
|
||||
"@deepseek-ai/dsh-terminal-bash": "workspace:^",
|
||||
"@deepseek-ai/dsh-pwsh-local": "workspace:^",
|
||||
|
||||
@@ -9,6 +9,7 @@ import { randomUUID } from 'node:crypto'
|
||||
import type { Context } from '@deepseek-ai/cordis'
|
||||
import z from '@deepseek-ai/schemastery'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
import { truncateWithoutSplittingSurrogatePair } from '@deepseek-ai/dsh-output-retention'
|
||||
import type { TerminalReadResult, TerminalSendResult, TerminalSessionId } from '@deepseek-ai/dsh-terminal'
|
||||
import { deadline, timeoutOf } from '@deepseek-ai/dsh-timeout'
|
||||
import { defineTool } from '@deepseek-ai/dsh-tools'
|
||||
@@ -58,7 +59,7 @@ function maybeTruncate(content: string, maxOutputChars: number, incomplete = fal
|
||||
if (content.length <= maxOutputChars && !incomplete) return content
|
||||
return content.length <= maxOutputChars
|
||||
? content + TRUNCATED_MESSAGE
|
||||
: content.slice(0, maxOutputChars) + TRUNCATED_MESSAGE
|
||||
: truncateWithoutSplittingSurrogatePair(content, maxOutputChars) + TRUNCATED_MESSAGE
|
||||
}
|
||||
|
||||
function markers(): CommandMarkers {
|
||||
|
||||
@@ -119,6 +119,7 @@ class StubTerminalSession implements TerminalBackendSession {
|
||||
pendingText = ''
|
||||
historyTruncated = false
|
||||
throwOnSend = false
|
||||
largeOutput = 'x'.repeat(100)
|
||||
|
||||
constructor(mode: StubMode) {
|
||||
this.mode = mode
|
||||
@@ -218,7 +219,7 @@ class StubTerminalSession implements TerminalBackendSession {
|
||||
return this.operation(Promise.resolve(this.result(output, 'stdin_read')))
|
||||
}
|
||||
const commandOutput = this.mode === 'large'
|
||||
? 'x'.repeat(100)
|
||||
? this.largeOutput
|
||||
: this.mode === 'nonzero' ? ''
|
||||
: this.mode === 'prompt-collision' ? this.motd
|
||||
: 'hello from stub'
|
||||
@@ -455,6 +456,21 @@ describe('tool-pwsh-persistent', () => {
|
||||
expect(stub.sessions[2]?.closed).toEqual(['external cleanup'])
|
||||
})
|
||||
|
||||
it('clips a split surrogate pair instead of leaving a lone half', async () => {
|
||||
const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 10 })
|
||||
await call(ctx, owner, 'warm up')
|
||||
const session = stub.sessions[0]!
|
||||
session.mode = 'large'
|
||||
// The tenth code unit is the emoji's high surrogate, so the clip drops the
|
||||
// unpaired half instead of emitting it.
|
||||
session.largeOutput = `${'x'.repeat(9)}😀tail`
|
||||
|
||||
const rendered = text(await call(ctx, owner, 'emoji'))
|
||||
|
||||
expect(rendered.startsWith(`${'x'.repeat(9)}<response clipped>`)).toBe(true)
|
||||
expect(rendered).not.toContain('\uD83D')
|
||||
})
|
||||
|
||||
it('waits for status digits after a torn completion marker', async () => {
|
||||
const { ctx, owner, stub } = await setup({ backendType: 'stub', maxOutputChars: 1_000 })
|
||||
await call(ctx, owner, 'warm up')
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
"references": [
|
||||
{ "path": "../../../vendor/cordis" },
|
||||
{ "path": "../../../vendor/schemastery" },
|
||||
{ "path": "../../util/output-retention" },
|
||||
{ "path": "../../core/agent" },
|
||||
{ "path": "../../core/tools" },
|
||||
{ "path": "../../terminal/terminal" },
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/util/output-retention/README.md
|
||||
README.md: bb0902b959f298f9f7b0a64de23d48e29bfd75ad
|
||||
README.zh.md: 7d96ba4da33a98b24eba53f9dd5a2f2c98b79f39
|
||||
README.md: ec16dd9626f2268a9a019373c3e135619e1ef2e5
|
||||
README.zh.md: 1ae37399224838d6b044dacc81a15983f82ff9c8
|
||||
@@ -9,7 +9,7 @@ English | [中文](README.zh.md)
|
||||
|
||||
## Summary
|
||||
|
||||
Use `dsh-output-retention` to cap the items or text a tool returns to a model while reporting what was omitted. `ItemRetainer` keeps an ordered head window and can report an exact omitted-item count; `TextRetainer` keeps head, tail, or head-and-tail byte windows without returning invalid UTF-8 cuts. `formatRetentionNotice` adds a consistent omission clause while each tool supplies its own recovery guidance. Grouping, line numbering, spill files, and provider errors remain tool responsibilities; consumers import this library directly rather than loading it through `cordis.yml`.
|
||||
Use `dsh-output-retention` to cap the items or text a tool returns to a model while reporting what was omitted. `ItemRetainer` keeps an ordered head window and can report an exact omitted-item count; `TextRetainer` keeps head, tail, or head-and-tail byte windows without returning invalid UTF-8 cuts. `formatRetentionNotice` adds a consistent omission clause while each tool supplies its own recovery guidance. `truncateWithoutSplittingSurrogatePair` caps a character-budget preview without leaving a lone high surrogate at the cut. Grouping, line numbering, spill files, and provider errors remain tool responsibilities; consumers import this library directly rather than loading it through `cordis.yml`.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
@@ -104,7 +104,7 @@ The library is built on one separation: it owns the mechanical question of what
|
||||
|
||||
| File | Role |
|
||||
|---|---|
|
||||
| [`src/index.ts`](src/index.ts) | `ItemRetainer`, `TextRetainer`, `describeOmitted`, and `formatRetentionNotice` |
|
||||
| [`src/index.ts`](src/index.ts) | `ItemRetainer`, `TextRetainer`, `describeOmitted`, `formatRetentionNotice`, and `truncateWithoutSplittingSurrogatePair` |
|
||||
| — | No runtime invariant companion is published; this pure utility owns no event stream or mutable runtime data; its value algebra is enforced by unit tests. |
|
||||
|
||||
### Two retainers, two resource models
|
||||
|
||||
@@ -9,7 +9,7 @@ kind: "package-library"
|
||||
|
||||
## 概述
|
||||
|
||||
使用 `dsh-output-retention` 限制工具返回给模型的项或文本量,并报告省略了什么。`ItemRetainer` 保留有序的头部窗口,并可报告精确的省略项数;`TextRetainer` 保留 head、tail 或 head-and-tail 字节窗口,且不会返回因切割而无效的 UTF-8。`formatRetentionNotice` 添加一致的省略子句,各工具则提供自己的恢复指引。分组、行号、spill 文件与提供方错误仍归工具负责;消费方直接导入本库,而不通过 `cordis.yml` 加载。
|
||||
使用 `dsh-output-retention` 限制工具返回给模型的项或文本量,并报告省略了什么。`ItemRetainer` 保留有序的头部窗口,并可报告精确的省略项数;`TextRetainer` 保留 head、tail 或 head-and-tail 字节窗口,且不会返回因切割而无效的 UTF-8。`formatRetentionNotice` 添加一致的省略子句,各工具则提供自己的恢复指引。`truncateWithoutSplittingSurrogatePair` 按字符预算截断预览时,不会在切口留下孤立的高位代理项。分组、行号、spill 文件与提供方错误仍归工具负责;消费方直接导入本库,而不通过 `cordis.yml` 加载。
|
||||
|
||||
## 目录
|
||||
|
||||
@@ -104,7 +104,7 @@ const footer = formatRetentionNotice(
|
||||
|
||||
| 文件 | 职责 |
|
||||
|---|---|
|
||||
| [`src/index.ts`](src/index.ts) | `ItemRetainer`、`TextRetainer`、`describeOmitted` 与 `formatRetentionNotice` |
|
||||
| [`src/index.ts`](src/index.ts) | `ItemRetainer`、`TextRetainer`、`describeOmitted`、`formatRetentionNotice` 与 `truncateWithoutSplittingSurrogatePair` |
|
||||
| — | 不发布运行时不变式伴生入口;这个纯工具不拥有事件流或可变运行时数据;其值代数由单元测试保证。 |
|
||||
|
||||
### 两个 retainer,两种资源模型
|
||||
|
||||
@@ -441,3 +441,20 @@ export function formatRetentionNotice(
|
||||
.filter(part => part.length > 0)
|
||||
.join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Cap `text` at `maxChars` UTF-16 code units without ending in a lone high
|
||||
* surrogate: a cut that lands inside a surrogate pair drops the unpaired half,
|
||||
* so the result is one code unit shorter than the cap. An unpaired surrogate is
|
||||
* not well-formed text — a strict JSON reader rejects a durable Session log that
|
||||
* carries one.
|
||||
*
|
||||
* @param text The text to cap.
|
||||
* @param maxChars The maximum number of UTF-16 code units to retain.
|
||||
* @returns A prefix of `text` no longer than `maxChars`, never ending in a lone high surrogate.
|
||||
*/
|
||||
export function truncateWithoutSplittingSurrogatePair(text: string, maxChars: number): string {
|
||||
if (text.length <= maxChars) return text
|
||||
const capped = text.slice(0, maxChars)
|
||||
return /[\uD800-\uDBFF]$/.test(capped) ? capped.slice(0, -1) : capped
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
type Omitted,
|
||||
type RetentionNotice,
|
||||
TextRetainer,
|
||||
truncateWithoutSplittingSurrogatePair,
|
||||
} from '@deepseek-ai/dsh-output-retention'
|
||||
|
||||
/** Decode a RetainedText via a round-trip helper for readable UTF-8 assertions. */
|
||||
@@ -374,3 +375,35 @@ describe('formatRetentionNotice', () => {
|
||||
expect(out).toBe('Omitted 500 bytes. Kept 2000B head + 2000B tail.')
|
||||
})
|
||||
})
|
||||
|
||||
describe('truncateWithoutSplittingSurrogatePair', () => {
|
||||
it('caps at maxChars UTF-16 code units', () => {
|
||||
expect(truncateWithoutSplittingSurrogatePair('abcdef', 3)).toBe('abc')
|
||||
expect(truncateWithoutSplittingSurrogatePair('abcdef', 0)).toBe('')
|
||||
expect(truncateWithoutSplittingSurrogatePair('abcdef', 1)).toBe('a')
|
||||
})
|
||||
|
||||
it('returns text within the cap unchanged', () => {
|
||||
expect(truncateWithoutSplittingSurrogatePair('abc', 3)).toBe('abc')
|
||||
expect(truncateWithoutSplittingSurrogatePair('abc', 9)).toBe('abc')
|
||||
expect(truncateWithoutSplittingSurrogatePair('', 0)).toBe('')
|
||||
})
|
||||
|
||||
it('drops the unpaired half when the cap lands inside a surrogate pair', () => {
|
||||
// '😀' is two code units, so these caps would otherwise keep its high surrogate alone.
|
||||
expect(truncateWithoutSplittingSurrogatePair('ab😀cd', 3)).toBe('ab')
|
||||
expect(truncateWithoutSplittingSurrogatePair('ab😀cd', 4)).toBe('ab😀')
|
||||
expect(truncateWithoutSplittingSurrogatePair('😀😀', 1)).toBe('')
|
||||
expect(truncateWithoutSplittingSurrogatePair('😀😀', 3)).toBe('😀')
|
||||
})
|
||||
|
||||
it('never ends in a lone high surrogate at any cap', () => {
|
||||
const text = 'a😀b😀c'
|
||||
for (let maxChars = 0; maxChars <= text.length; maxChars++) {
|
||||
const capped = truncateWithoutSplittingSurrogatePair(text, maxChars)
|
||||
expect(capped.length).toBeLessThanOrEqual(maxChars)
|
||||
expect(capped).not.toMatch(/[\uD800-\uDBFF]$/)
|
||||
expect(capped).toBe(text.slice(0, capped.length))
|
||||
}
|
||||
})
|
||||
})
|
||||
Generated
+9
@@ -7996,6 +7996,9 @@ importers:
|
||||
'@deepseek-ai/dsh-llm':
|
||||
specifier: workspace:^
|
||||
version: link:../../llm/llm
|
||||
'@deepseek-ai/dsh-output-retention':
|
||||
specifier: workspace:^
|
||||
version: link:../../util/output-retention
|
||||
'@deepseek-ai/dsh-sandbox':
|
||||
specifier: workspace:^
|
||||
version: link:../../sandbox/sandbox
|
||||
@@ -10792,6 +10795,9 @@ importers:
|
||||
'@deepseek-ai/dsh-llm':
|
||||
specifier: workspace:^
|
||||
version: link:../../llm/llm
|
||||
'@deepseek-ai/dsh-output-retention':
|
||||
specifier: workspace:^
|
||||
version: link:../../util/output-retention
|
||||
'@deepseek-ai/dsh-sandbox':
|
||||
specifier: workspace:^
|
||||
version: link:../../sandbox/sandbox
|
||||
@@ -10914,6 +10920,9 @@ importers:
|
||||
'@deepseek-ai/dsh-llm':
|
||||
specifier: workspace:^
|
||||
version: link:../../llm/llm
|
||||
'@deepseek-ai/dsh-output-retention':
|
||||
specifier: workspace:^
|
||||
version: link:../../util/output-retention
|
||||
'@deepseek-ai/dsh-pwsh-local':
|
||||
specifier: workspace:^
|
||||
version: link:../pwsh-local
|
||||
|
||||
Reference in new issue
Block a user