diff --git a/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.i18n.yaml index 2e677758f6..ccb175f6b5 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.i18n.yaml @@ -9,8 +9,8 @@ en: 064ec0081b5c2946 zh: 4b67a513ea949220 /explicit-output-in-typescript-ptc-programs/change: - en: 6afdaebc78b2c265 - zh: 8744ecc12707d8a6 + en: 6bf15e66fa7c5b1f + zh: 56337c27da649c04 /explicit-output-in-typescript-ptc-programs/migration: en: e7e79e2077eaae80 zh: f34abc7fe3f7c764 diff --git a/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.md index c0ba88a113..bd8fc24f2a 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.md @@ -13,7 +13,7 @@ TypeScript `run_code` programs display selected values through `inspect(...value Only CUA screenshots and `read_image` return ordinary `Image` values in TypeScript programs. Their `path` points to an attachment file readable in the execution environment; `width` and `height` describe its stored pixels. Pass selected images to `inspect` alongside explanatory text, including images nested in arrays or objects. Read CUA window IDs from the window object’s `windowId`. -Other image-producing tools retain their declared program values; their pictures are not automatically attached in TypeScript PTC. Image adaptation requires a readable attachment path in the program filesystem. `fs-ssh` does not provide this mapping, so `read_image` and CUA screenshot bindings fail in that composition. Use a local filesystem composition for program Images. +Other image-producing tools retain their declared program values and automatically attach their pictures to the model context. Image adaptation requires a readable attachment path in the program filesystem. `fs-ssh` does not provide this mapping, so `read_image` and CUA screenshot bindings fail in that composition. Use a local filesystem composition for program Images. ## Migration diff --git a/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.zh.md index 6e679c87b3..0ebe300022 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/ptc-inspect-output/guide.zh.md @@ -13,7 +13,7 @@ TypeScript `run_code` 程序通过 `inspect(...values)` 展示选定的值。此 只有 CUA 截图与 `read_image` 在 TypeScript 程序中返回普通 `Image`,其 `path` 指向执行环境可读取的附件文件,`width` 与 `height` 为该文件的像素尺寸。使用 `inspect` 将选中的图片与说明一起展示;图片可嵌入数组或对象。CUA 窗口 ID 从窗口对象的 `windowId` 读取。 -其他图片工具保留其声明的程序返回值;TypeScript PTC 不自动附加它们的图片。图片适配要求附件在程序文件系统中有可读路径。`fs-ssh` 未提供这项映射,因此该组合中的 `read_image` 与 CUA 截图绑定会失败。需要程序 Image 时,使用本地文件系统组合。 +其他图片工具保留其声明的程序返回值,并将图片自动附加到模型上下文。图片适配要求附件在程序文件系统中有可读路径。`fs-ssh` 未提供这项映射,因此该组合中的 `read_image` 与 CUA 截图绑定会失败。需要程序 Image 时,使用本地文件系统组合。 ## 迁移 diff --git a/packages/core/tool-executor-ptc/README.i18n.yaml b/packages/core/tool-executor-ptc/README.i18n.yaml index 27cf2e78e0..668d8072db 100644 --- a/packages/core/tool-executor-ptc/README.i18n.yaml +++ b/packages/core/tool-executor-ptc/README.i18n.yaml @@ -48,8 +48,8 @@ en: 4bf1cd2890cfe6c4 zh: 5b230b0adc09ddfb /deepseek-ai-dsh-tool-executor-ptc/known-limitations-and-deferred-work: - en: 8faaf0795f8be5bc - zh: 538710004f7abff0 + en: 9af95ec65871e278 + zh: f8df79e069be1c50 /deepseek-ai-dsh-tool-executor-ptc/known-limitations-and-deferred-work/dev-note: en: e67f633dc6fe7773 zh: 3bb0c20c93f16f98 diff --git a/packages/core/tool-executor-ptc/README.md b/packages/core/tool-executor-ptc/README.md index 0cb2a124dc..cfba8f2cef 100644 --- a/packages/core/tool-executor-ptc/README.md +++ b/packages/core/tool-executor-ptc/README.md @@ -132,7 +132,7 @@ Prefix-stable while the PTC mode selection, generated SDK, transport schema, and These constraints apply to the host integration and accepted calls. -- **Program Images cover read_image and CUA screenshots** — other image-producing tools keep their declared values, but TypeScript PTC does not automatically forward their pictures. +- **Program Images cover read_image and CUA screenshots** — other image-producing tools keep their declared values and automatically attach their pictures as additional model context. - **Program Images require attachment path mapping** — the filesystem must map stored host attachments to readable program paths. `fs-ssh` has no such mapping, so image adaptation rejects in that composition. - **No built-in reset control** — the shipped Loader plugin does not expose its executor instance through a service, GUI control or SDK endpoint. Hosts needing reset must mount and retain `PtcExecutor` in their own composition; that integration owns retries after a failed reset. diff --git a/packages/core/tool-executor-ptc/README.zh.md b/packages/core/tool-executor-ptc/README.zh.md index 511cebd05b..78d0cb835f 100644 --- a/packages/core/tool-executor-ptc/README.zh.md +++ b/packages/core/tool-executor-ptc/README.zh.md @@ -132,7 +132,7 @@ Error: code run failed (abort): run_code exceeded its limit of 2048 tool calls ( 以下约束适用于宿主集成及已开始的调用。 -- **程序 Image 仅覆盖 read_image 与 CUA 截图**:其他图片工具保留其声明的返回值,但 TypeScript PTC 不自动转发它们的图片。 +- **程序 Image 仅覆盖 read_image 与 CUA 截图**:其他图片工具保留其声明的返回值,并将图片自动附加到模型的额外上下文。 - **程序 Image 要求附件路径映射**:文件系统必须将宿主存储的附件映射为程序可读的路径。`fs-ssh` 没有这项映射,因此该组合中的图片适配会拒绝。 - **没有内置重置控件**:自带的 Loader 插件不通过服务、GUI 控件或 SDK 端点暴露执行器实例。需要重置的宿主必须在自己的组合中挂载并保留 `PtcExecutor`,该集成负责失败后的重试。 diff --git a/packages/core/tool-executor-ptc/src/ptc.ts b/packages/core/tool-executor-ptc/src/ptc.ts index 0ea10fb8b0..b9d452e5ef 100644 --- a/packages/core/tool-executor-ptc/src/ptc.ts +++ b/packages/core/tool-executor-ptc/src/ptc.ts @@ -539,6 +539,8 @@ export function createRunCodeTool( const binding = (schema: ToolSchema): PtcBindingFunction => async (rawArgs: unknown): Promise => { const { name } = schema + // FIXME: Let image-producing tools declare their program result adaptation. + const usesProgramImage = runtime.language === 'typescript' && (name === 'read_image' || name === 'cua') if (runOver()) { throw new Error(`run_code run is over (${String(runController.signal.reason)}); ${name} not dispatched`) } @@ -644,7 +646,7 @@ export function createRunCodeTool( /* v8 ignore next -- commit() runs only after `settled` flipped, which set parked. */ if (parked === undefined) return const result = await executor.complete(parked.exec, parked) - if (runtime.language !== 'typescript' && !result.isError && result.content.some(block => block.type === 'image')) { + if (!usesProgramImage && !result.isError && result.content.some(block => block.type === 'image')) { exec.deferContext(createUserMessage({ content: result.content, source: { kind: 'ptc-mode' }, @@ -682,8 +684,7 @@ export function createRunCodeTool( // only the binding name. Native content and internal error metadata // stay outside the program-facing failure contract. if (outcome.isError) throw new Error(outcome.message) - // FIXME: Let image-producing tools declare their program result adaptation. - if (runtime.language === 'typescript' && (name === 'read_image' || name === 'cua')) { + if (usesProgramImage) { const images = outcome.content.filter(block => block.type === 'image') const first = images[0] if (first !== undefined && images.length === 1) return programImage(first.attachment) diff --git a/packages/core/tool-executor-ptc/tests/ptc.spec.ts b/packages/core/tool-executor-ptc/tests/ptc.spec.ts index 69ef4fc423..ecdea292c1 100644 --- a/packages/core/tool-executor-ptc/tests/ptc.spec.ts +++ b/packages/core/tool-executor-ptc/tests/ptc.spec.ts @@ -1675,8 +1675,7 @@ describe('the run_code dispatch bridge', () => { const result = await runCode(ctx, 'program') - if (language === 'typescript') expect(result.additionalContexts).toBeUndefined() - else expect(result.additionalContexts).toMatchObject([{ + expect(result.additionalContexts).toMatchObject([{ source: { kind: 'ptc-mode' }, content: [ { type: 'text', text: 'image result' }, diff --git a/packages/document/tool-office/tests/tool-office.spec.ts b/packages/document/tool-office/tests/tool-office.spec.ts index 81be68d2e7..c52338b665 100644 --- a/packages/document/tool-office/tests/tool-office.spec.ts +++ b/packages/document/tool-office/tests/tool-office.spec.ts @@ -668,7 +668,10 @@ describe('PTC dispatch', () => { const dispatch = agent.session.snapshotEvents().find(event => event.type === 'tool/ptc-dispatch') expect(dispatch?.data.content.some(block => block.type === 'image')).toBe(true) expect(result.content.some(block => block.type === 'image')).toBe(false) - expect(result.additionalContexts).toBeUndefined() + expect(result.additionalContexts).toHaveLength(1) + expect(result.additionalContexts?.[0]).toMatchObject({ + source: { kind: 'ptc-mode' }, content: dispatch?.data.content, + }) const denied: string[] = [] ctx.on('tools/pre-execute', (exec, next) => { diff --git a/packages/spill/spill-policy/tests/multimodal-recovery.spec.ts b/packages/spill/spill-policy/tests/multimodal-recovery.spec.ts index 531af3ef14..bb968f0357 100644 --- a/packages/spill/spill-policy/tests/multimodal-recovery.spec.ts +++ b/packages/spill/spill-policy/tests/multimodal-recovery.spec.ts @@ -225,25 +225,19 @@ describe('multimodal recovery through real providers', () => { : '[await tools.ends({}), await tools.middle({}), await tools.small({})]' const result = await execute('run_code', { code: `const [ends, middle, small] = ${calls}; -inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length })); -const fs = await import('node:fs/promises'); -for (const [index, image] of [ends.content[1], ends.content[3]].entries()) { - const path = 'recovered-' + index + '.png'; - await fs.writeFile(path, Buffer.from(image.data, 'base64')); - inspect(await tools.read_image({ file_path: path })); -}`, +inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length }));`, description: 'Inspect independent image result budgets', }) expect(result.isError).toBe(false) expect(result.value).toMatchObject({ logs: [JSON.stringify({ ends: LONG.length, middle: LONG.length, small: SMALL.length })] }) const dispatches = session.snapshotEvents().filter(event => event.type === 'tool/ptc-dispatch').map(event => event.data) - expect(dispatches).toHaveLength(5) + expect(dispatches).toHaveLength(3) for (const dispatch of dispatches) expect(cost(dispatch.content)).toBeLessThanOrEqual(CAP) expect(dispatches.find(event => event.name === 'ends')?.content.filter(block => block.type === 'image')).toHaveLength(2) expect(textOf(dispatches.find(event => event.name === 'middle')!.content)).toContain('Omitted 1 images.') expect(dispatches.find(event => event.name === 'small')?.content).toEqual([text(SMALL)]) - expect(imagesOf(result.content)).toHaveLength(2) - expect(imagesOf(result.additionalContexts?.flatMap(context => context.content) ?? [])).toEqual([]) + expect(imagesOf(result.content)).toEqual([]) + expect(imagesOf(result.additionalContexts?.flatMap(context => context.content) ?? [])).toHaveLength(2) expect(await requestImages(result.content, result.additionalContexts)).toEqual(['red', 'blue']) }) })