fix(ptc): preserve automatic images for unadapted tools

This commit is contained in:
_Kerman committed 2026-11-09 21:46:06 +08:00
1 parent 8566fdb439
commit 5bcee95efe
10 files changed
+21 -24

No files matched your search

@@ -9,8 +9,8 @@
en: 064ec0081b5c2946
zh: 4b67a513ea949220
/explicit-output-in-typescript-ptc-programs/change:
en: 6afdaebc78b2c265
zh: 8744ecc12707d8a6
en: 6bf15e66fa7c5b1f
zh: 56337c27da649c04
/explicit-output-in-typescript-ptc-programs/migration:
en: e7e79e2077eaae80
zh: f34abc7fe3f7c764
@@ -13,7 +13,7 @@ TypeScript `run_code` programs display selected values through `inspect(...value
Only CUA screenshots and `read_image` return ordinary `Image` values in TypeScript programs. Their `path` points to an attachment file readable in the execution environment; `width` and `height` describe its stored pixels. Pass selected images to `inspect` alongside explanatory text, including images nested in arrays or objects. Read CUA window IDs from the window object’s `windowId`.
Other image-producing tools retain their declared program values; their pictures are not automatically attached in TypeScript PTC. Image adaptation requires a readable attachment path in the program filesystem. `fs-ssh` does not provide this mapping, so `read_image` and CUA screenshot bindings fail in that composition. Use a local filesystem composition for program Images.
Other image-producing tools retain their declared program values and automatically attach their pictures to the model context. Image adaptation requires a readable attachment path in the program filesystem. `fs-ssh` does not provide this mapping, so `read_image` and CUA screenshot bindings fail in that composition. Use a local filesystem composition for program Images.
## Migration
@@ -13,7 +13,7 @@ TypeScript `run_code` 程序通过 `inspect(...values)` 展示选定的值。此
只有 CUA 截图与 `read_image` 在 TypeScript 程序中返回普通 `Image`,其 `path` 指向执行环境可读取的附件文件,`width` 与 `height` 为该文件的像素尺寸。使用 `inspect` 将选中的图片与说明一起展示;图片可嵌入数组或对象。CUA 窗口 ID 从窗口对象的 `windowId` 读取。
其他图片工具保留其声明的程序返回值;TypeScript PTC 不自动附加它们的图片。图片适配要求附件在程序文件系统中有可读路径。`fs-ssh` 未提供这项映射,因此该组合中的 `read_image` 与 CUA 截图绑定会失败。需要程序 Image 时,使用本地文件系统组合。
其他图片工具保留其声明的程序返回值,并将图片自动附加到模型上下文。图片适配要求附件在程序文件系统中有可读路径。`fs-ssh` 未提供这项映射,因此该组合中的 `read_image` 与 CUA 截图绑定会失败。需要程序 Image 时,使用本地文件系统组合。
## 迁移
@@ -48,8 +48,8 @@
en: 4bf1cd2890cfe6c4
zh: 5b230b0adc09ddfb
/deepseek-ai-dsh-tool-executor-ptc/known-limitations-and-deferred-work:
en: 8faaf0795f8be5bc
zh: 538710004f7abff0
en: 9af95ec65871e278
zh: f8df79e069be1c50
/deepseek-ai-dsh-tool-executor-ptc/known-limitations-and-deferred-work/dev-note:
en: e67f633dc6fe7773
zh: 3bb0c20c93f16f98
+1 -1
View File
@@ -132,7 +132,7 @@ Prefix-stable while the PTC mode selection, generated SDK, transport schema, and
These constraints apply to the host integration and accepted calls.
- **Program Images cover read_image and CUA screenshots** — other image-producing tools keep their declared values, but TypeScript PTC does not automatically forward their pictures.
- **Program Images cover read_image and CUA screenshots** — other image-producing tools keep their declared values and automatically attach their pictures as additional model context.
- **Program Images require attachment path mapping** — the filesystem must map stored host attachments to readable program paths. `fs-ssh` has no such mapping, so image adaptation rejects in that composition.
- **No built-in reset control** — the shipped Loader plugin does not expose its executor instance through a service, GUI control or SDK endpoint. Hosts needing reset must mount and retain `PtcExecutor` in their own composition; that integration owns retries after a failed reset.
+1 -1
View File
@@ -132,7 +132,7 @@ Error: code run failed (abort): run_code exceeded its limit of 2048 tool calls (
以下约束适用于宿主集成及已开始的调用。
- **程序 Image 仅覆盖 read_image 与 CUA 截图**:其他图片工具保留其声明的返回值,但 TypeScript PTC 不自动转发它们的图片。
- **程序 Image 仅覆盖 read_image 与 CUA 截图**:其他图片工具保留其声明的返回值,并将图片自动附加到模型的额外上下文。
- **程序 Image 要求附件路径映射**:文件系统必须将宿主存储的附件映射为程序可读的路径。`fs-ssh` 没有这项映射,因此该组合中的图片适配会拒绝。
- **没有内置重置控件**:自带的 Loader 插件不通过服务、GUI 控件或 SDK 端点暴露执行器实例。需要重置的宿主必须在自己的组合中挂载并保留 `PtcExecutor`,该集成负责失败后的重试。
+4 -3
View File
@@ -539,6 +539,8 @@ export function createRunCodeTool(
const binding = (schema: ToolSchema): PtcBindingFunction => async (rawArgs: unknown): Promise<JsonValue> => {
const { name } = schema
// FIXME: Let image-producing tools declare their program result adaptation.
const usesProgramImage = runtime.language === 'typescript' && (name === 'read_image' || name === 'cua')
if (runOver()) {
throw new Error(`run_code run is over (${String(runController.signal.reason)}); ${name} not dispatched`)
}
@@ -644,7 +646,7 @@ export function createRunCodeTool(
/* v8 ignore next -- commit() runs only after `settled` flipped, which set parked. */
if (parked === undefined) return
const result = await executor.complete(parked.exec, parked)
if (runtime.language !== 'typescript' && !result.isError && result.content.some(block => block.type === 'image')) {
if (!usesProgramImage && !result.isError && result.content.some(block => block.type === 'image')) {
exec.deferContext(createUserMessage({
content: result.content,
source: { kind: 'ptc-mode' },
@@ -682,8 +684,7 @@ export function createRunCodeTool(
// only the binding name. Native content and internal error metadata
// stay outside the program-facing failure contract.
if (outcome.isError) throw new Error(outcome.message)
// FIXME: Let image-producing tools declare their program result adaptation.
if (runtime.language === 'typescript' && (name === 'read_image' || name === 'cua')) {
if (usesProgramImage) {
const images = outcome.content.filter(block => block.type === 'image')
const first = images[0]
if (first !== undefined && images.length === 1) return programImage(first.attachment)
@@ -1675,8 +1675,7 @@ describe('the run_code dispatch bridge', () => {
const result = await runCode(ctx, 'program')
if (language === 'typescript') expect(result.additionalContexts).toBeUndefined()
else expect(result.additionalContexts).toMatchObject([{
expect(result.additionalContexts).toMatchObject([{
source: { kind: 'ptc-mode' },
content: [
{ type: 'text', text: 'image result' },
@@ -668,7 +668,10 @@ describe('PTC dispatch', () => {
const dispatch = agent.session.snapshotEvents().find(event => event.type === 'tool/ptc-dispatch')
expect(dispatch?.data.content.some(block => block.type === 'image')).toBe(true)
expect(result.content.some(block => block.type === 'image')).toBe(false)
expect(result.additionalContexts).toBeUndefined()
expect(result.additionalContexts).toHaveLength(1)
expect(result.additionalContexts?.[0]).toMatchObject({
source: { kind: 'ptc-mode' }, content: dispatch?.data.content,
})
const denied: string[] = []
ctx.on('tools/pre-execute', (exec, next) => {
@@ -225,25 +225,19 @@ describe('multimodal recovery through real providers', () => {
: '[await tools.ends({}), await tools.middle({}), await tools.small({})]'
const result = await execute('run_code', {
code: `const [ends, middle, small] = ${calls};
inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length }));
const fs = await import('node:fs/promises');
for (const [index, image] of [ends.content[1], ends.content[3]].entries()) {
const path = 'recovered-' + index + '.png';
await fs.writeFile(path, Buffer.from(image.data, 'base64'));
inspect(await tools.read_image({ file_path: path }));
}`,
inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length }));`,
description: 'Inspect independent image result budgets',
})
expect(result.isError).toBe(false)
expect(result.value).toMatchObject({ logs: [JSON.stringify({ ends: LONG.length, middle: LONG.length, small: SMALL.length })] })
const dispatches = session.snapshotEvents().filter(event => event.type === 'tool/ptc-dispatch').map(event => event.data)
expect(dispatches).toHaveLength(5)
expect(dispatches).toHaveLength(3)
for (const dispatch of dispatches) expect(cost(dispatch.content)).toBeLessThanOrEqual(CAP)
expect(dispatches.find(event => event.name === 'ends')?.content.filter(block => block.type === 'image')).toHaveLength(2)
expect(textOf(dispatches.find(event => event.name === 'middle')!.content)).toContain('Omitted 1 images.')
expect(dispatches.find(event => event.name === 'small')?.content).toEqual([text(SMALL)])
expect(imagesOf(result.content)).toHaveLength(2)
expect(imagesOf(result.additionalContexts?.flatMap(context => context.content) ?? [])).toEqual([])
expect(imagesOf(result.content)).toEqual([])
expect(imagesOf(result.additionalContexts?.flatMap(context => context.content) ?? [])).toHaveLength(2)
expect(await requestImages(result.content, result.additionalContexts)).toEqual(['red', 'blue'])
})
})