fix(ptc): preserve automatic images for unadapted tools
This commit is contained in:
1 parent
8566fdb439
commit
5bcee95efe
10 files changed
+21
-24
No files matched your search
@@ -9,8 +9,8 @@
|
||||
en: 064ec0081b5c2946
|
||||
zh: 4b67a513ea949220
|
||||
/explicit-output-in-typescript-ptc-programs/change:
|
||||
en: 6afdaebc78b2c265
|
||||
zh: 8744ecc12707d8a6
|
||||
en: 6bf15e66fa7c5b1f
|
||||
zh: 56337c27da649c04
|
||||
/explicit-output-in-typescript-ptc-programs/migration:
|
||||
en: e7e79e2077eaae80
|
||||
zh: f34abc7fe3f7c764
|
||||
@@ -13,7 +13,7 @@ TypeScript `run_code` programs display selected values through `inspect(...value
|
||||
|
||||
Only CUA screenshots and `read_image` return ordinary `Image` values in TypeScript programs. Their `path` points to an attachment file readable in the execution environment; `width` and `height` describe its stored pixels. Pass selected images to `inspect` alongside explanatory text, including images nested in arrays or objects. Read CUA window IDs from the window object’s `windowId`.
|
||||
|
||||
Other image-producing tools retain their declared program values; their pictures are not automatically attached in TypeScript PTC. Image adaptation requires a readable attachment path in the program filesystem. `fs-ssh` does not provide this mapping, so `read_image` and CUA screenshot bindings fail in that composition. Use a local filesystem composition for program Images.
|
||||
Other image-producing tools retain their declared program values and automatically attach their pictures to the model context. Image adaptation requires a readable attachment path in the program filesystem. `fs-ssh` does not provide this mapping, so `read_image` and CUA screenshot bindings fail in that composition. Use a local filesystem composition for program Images.
|
||||
|
||||
## Migration
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ TypeScript `run_code` 程序通过 `inspect(...values)` 展示选定的值。此
|
||||
|
||||
只有 CUA 截图与 `read_image` 在 TypeScript 程序中返回普通 `Image`,其 `path` 指向执行环境可读取的附件文件,`width` 与 `height` 为该文件的像素尺寸。使用 `inspect` 将选中的图片与说明一起展示;图片可嵌入数组或对象。CUA 窗口 ID 从窗口对象的 `windowId` 读取。
|
||||
|
||||
其他图片工具保留其声明的程序返回值;TypeScript PTC 不自动附加它们的图片。图片适配要求附件在程序文件系统中有可读路径。`fs-ssh` 未提供这项映射,因此该组合中的 `read_image` 与 CUA 截图绑定会失败。需要程序 Image 时,使用本地文件系统组合。
|
||||
其他图片工具保留其声明的程序返回值,并将图片自动附加到模型上下文。图片适配要求附件在程序文件系统中有可读路径。`fs-ssh` 未提供这项映射,因此该组合中的 `read_image` 与 CUA 截图绑定会失败。需要程序 Image 时,使用本地文件系统组合。
|
||||
|
||||
## 迁移
|
||||
|
||||
|
||||
@@ -48,8 +48,8 @@
|
||||
en: 4bf1cd2890cfe6c4
|
||||
zh: 5b230b0adc09ddfb
|
||||
/deepseek-ai-dsh-tool-executor-ptc/known-limitations-and-deferred-work:
|
||||
en: 8faaf0795f8be5bc
|
||||
zh: 538710004f7abff0
|
||||
en: 9af95ec65871e278
|
||||
zh: f8df79e069be1c50
|
||||
/deepseek-ai-dsh-tool-executor-ptc/known-limitations-and-deferred-work/dev-note:
|
||||
en: e67f633dc6fe7773
|
||||
zh: 3bb0c20c93f16f98
|
||||
@@ -132,7 +132,7 @@ Prefix-stable while the PTC mode selection, generated SDK, transport schema, and
|
||||
|
||||
These constraints apply to the host integration and accepted calls.
|
||||
|
||||
- **Program Images cover read_image and CUA screenshots** — other image-producing tools keep their declared values, but TypeScript PTC does not automatically forward their pictures.
|
||||
- **Program Images cover read_image and CUA screenshots** — other image-producing tools keep their declared values and automatically attach their pictures as additional model context.
|
||||
- **Program Images require attachment path mapping** — the filesystem must map stored host attachments to readable program paths. `fs-ssh` has no such mapping, so image adaptation rejects in that composition.
|
||||
|
||||
- **No built-in reset control** — the shipped Loader plugin does not expose its executor instance through a service, GUI control or SDK endpoint. Hosts needing reset must mount and retain `PtcExecutor` in their own composition; that integration owns retries after a failed reset.
|
||||
|
||||
@@ -132,7 +132,7 @@ Error: code run failed (abort): run_code exceeded its limit of 2048 tool calls (
|
||||
|
||||
以下约束适用于宿主集成及已开始的调用。
|
||||
|
||||
- **程序 Image 仅覆盖 read_image 与 CUA 截图**:其他图片工具保留其声明的返回值,但 TypeScript PTC 不自动转发它们的图片。
|
||||
- **程序 Image 仅覆盖 read_image 与 CUA 截图**:其他图片工具保留其声明的返回值,并将图片自动附加到模型的额外上下文。
|
||||
- **程序 Image 要求附件路径映射**:文件系统必须将宿主存储的附件映射为程序可读的路径。`fs-ssh` 没有这项映射,因此该组合中的图片适配会拒绝。
|
||||
|
||||
- **没有内置重置控件**:自带的 Loader 插件不通过服务、GUI 控件或 SDK 端点暴露执行器实例。需要重置的宿主必须在自己的组合中挂载并保留 `PtcExecutor`,该集成负责失败后的重试。
|
||||
|
||||
@@ -539,6 +539,8 @@ export function createRunCodeTool(
|
||||
|
||||
const binding = (schema: ToolSchema): PtcBindingFunction => async (rawArgs: unknown): Promise<JsonValue> => {
|
||||
const { name } = schema
|
||||
// FIXME: Let image-producing tools declare their program result adaptation.
|
||||
const usesProgramImage = runtime.language === 'typescript' && (name === 'read_image' || name === 'cua')
|
||||
if (runOver()) {
|
||||
throw new Error(`run_code run is over (${String(runController.signal.reason)}); ${name} not dispatched`)
|
||||
}
|
||||
@@ -644,7 +646,7 @@ export function createRunCodeTool(
|
||||
/* v8 ignore next -- commit() runs only after `settled` flipped, which set parked. */
|
||||
if (parked === undefined) return
|
||||
const result = await executor.complete(parked.exec, parked)
|
||||
if (runtime.language !== 'typescript' && !result.isError && result.content.some(block => block.type === 'image')) {
|
||||
if (!usesProgramImage && !result.isError && result.content.some(block => block.type === 'image')) {
|
||||
exec.deferContext(createUserMessage({
|
||||
content: result.content,
|
||||
source: { kind: 'ptc-mode' },
|
||||
@@ -682,8 +684,7 @@ export function createRunCodeTool(
|
||||
// only the binding name. Native content and internal error metadata
|
||||
// stay outside the program-facing failure contract.
|
||||
if (outcome.isError) throw new Error(outcome.message)
|
||||
// FIXME: Let image-producing tools declare their program result adaptation.
|
||||
if (runtime.language === 'typescript' && (name === 'read_image' || name === 'cua')) {
|
||||
if (usesProgramImage) {
|
||||
const images = outcome.content.filter(block => block.type === 'image')
|
||||
const first = images[0]
|
||||
if (first !== undefined && images.length === 1) return programImage(first.attachment)
|
||||
|
||||
@@ -1675,8 +1675,7 @@ describe('the run_code dispatch bridge', () => {
|
||||
|
||||
const result = await runCode(ctx, 'program')
|
||||
|
||||
if (language === 'typescript') expect(result.additionalContexts).toBeUndefined()
|
||||
else expect(result.additionalContexts).toMatchObject([{
|
||||
expect(result.additionalContexts).toMatchObject([{
|
||||
source: { kind: 'ptc-mode' },
|
||||
content: [
|
||||
{ type: 'text', text: 'image result' },
|
||||
|
||||
@@ -668,7 +668,10 @@ describe('PTC dispatch', () => {
|
||||
const dispatch = agent.session.snapshotEvents().find(event => event.type === 'tool/ptc-dispatch')
|
||||
expect(dispatch?.data.content.some(block => block.type === 'image')).toBe(true)
|
||||
expect(result.content.some(block => block.type === 'image')).toBe(false)
|
||||
expect(result.additionalContexts).toBeUndefined()
|
||||
expect(result.additionalContexts).toHaveLength(1)
|
||||
expect(result.additionalContexts?.[0]).toMatchObject({
|
||||
source: { kind: 'ptc-mode' }, content: dispatch?.data.content,
|
||||
})
|
||||
|
||||
const denied: string[] = []
|
||||
ctx.on('tools/pre-execute', (exec, next) => {
|
||||
|
||||
@@ -225,25 +225,19 @@ describe('multimodal recovery through real providers', () => {
|
||||
: '[await tools.ends({}), await tools.middle({}), await tools.small({})]'
|
||||
const result = await execute('run_code', {
|
||||
code: `const [ends, middle, small] = ${calls};
|
||||
inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length }));
|
||||
const fs = await import('node:fs/promises');
|
||||
for (const [index, image] of [ends.content[1], ends.content[3]].entries()) {
|
||||
const path = 'recovered-' + index + '.png';
|
||||
await fs.writeFile(path, Buffer.from(image.data, 'base64'));
|
||||
inspect(await tools.read_image({ file_path: path }));
|
||||
}`,
|
||||
inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length }));`,
|
||||
description: 'Inspect independent image result budgets',
|
||||
})
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.value).toMatchObject({ logs: [JSON.stringify({ ends: LONG.length, middle: LONG.length, small: SMALL.length })] })
|
||||
const dispatches = session.snapshotEvents().filter(event => event.type === 'tool/ptc-dispatch').map(event => event.data)
|
||||
expect(dispatches).toHaveLength(5)
|
||||
expect(dispatches).toHaveLength(3)
|
||||
for (const dispatch of dispatches) expect(cost(dispatch.content)).toBeLessThanOrEqual(CAP)
|
||||
expect(dispatches.find(event => event.name === 'ends')?.content.filter(block => block.type === 'image')).toHaveLength(2)
|
||||
expect(textOf(dispatches.find(event => event.name === 'middle')!.content)).toContain('Omitted 1 images.')
|
||||
expect(dispatches.find(event => event.name === 'small')?.content).toEqual([text(SMALL)])
|
||||
expect(imagesOf(result.content)).toHaveLength(2)
|
||||
expect(imagesOf(result.additionalContexts?.flatMap(context => context.content) ?? [])).toEqual([])
|
||||
expect(imagesOf(result.content)).toEqual([])
|
||||
expect(imagesOf(result.additionalContexts?.flatMap(context => context.content) ?? [])).toHaveLength(2)
|
||||
expect(await requestImages(result.content, result.additionalContexts)).toEqual(['red', 'blue'])
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user