diff --git a/packages/core/tool-executor-ptc/src/ptc.ts b/packages/core/tool-executor-ptc/src/ptc.ts index 4b13f5d7f1..27a9b269ac 100644 --- a/packages/core/tool-executor-ptc/src/ptc.ts +++ b/packages/core/tool-executor-ptc/src/ptc.ts @@ -679,7 +679,7 @@ export function createRunCodeTool( if (runtime.language === 'typescript' && (name === 'read_image' || name === 'cua')) { const images = outcome.content.filter(block => block.type === 'image') const first = images[0] - if (images.length === 1 && first !== undefined) return programImage(first.attachment) + if (first !== undefined && images.length === 1) return programImage(first.attachment) if (name === 'read_image' || images.length > 1) throw new Error('Image-producing tools must return exactly one image') } return outcome.value diff --git a/packages/core/tool-executor-ptc/tests/ptc.spec.ts b/packages/core/tool-executor-ptc/tests/ptc.spec.ts index 8d92c438ec..3417a82c33 100644 --- a/packages/core/tool-executor-ptc/tests/ptc.spec.ts +++ b/packages/core/tool-executor-ptc/tests/ptc.spec.ts @@ -1651,8 +1651,8 @@ describe('the run_code dispatch bridge', () => { ]) }) - it('keeps image-bearing sub-call results out of the conversation until inspected', async () => { - const { ctx, runtime } = await setup({ mode: 'ptc' }) + it.each(['typescript', 'python'])('records image-bearing sub-call results for a %s runtime', async (language) => { + const { ctx, runtime } = await setup({ mode: 'ptc', runtime: { language } }) ctx.tools.register(defineContentToolFixture({ name: 'image_result', description: 'Return one durable image.', @@ -1675,10 +1675,59 @@ describe('the run_code dispatch bridge', () => { const result = await runCode(ctx, 'program') - expect(result.additionalContexts).toBeUndefined() + if (language === 'typescript') expect(result.additionalContexts).toBeUndefined() + else expect(result.additionalContexts).toMatchObject([{ + source: { kind: 'ptc-mode' }, + content: [ + { type: 'text', text: 'image result' }, + { type: 'image', attachment: { attachmentId: 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' } }, + ], + }]) expect(result.content).toEqual([{ type: 'text', text: 'done' }]) }) + it.each([ + { name: 'read_image', count: 0, message: 'Image-producing tools must return exactly one image' }, + { name: 'read_image', count: 2, message: 'Image-producing tools must return exactly one image' }, + { name: 'cua', count: 2, message: 'Image-producing tools must return exactly one image' }, + { name: 'read_image', count: 1, message: 'Image storage must expose a readable path in the program filesystem' }, + ])('rejects $name with $count images when its program image is unavailable', async ({ name, count, message }) => { + const { ctx, runtime } = await setup({ mode: 'ptc' }) + onTestFinished(() => ctx.fiber.dispose()) + ctx.tools.register(defineContentToolFixture({ + name, description: 'Return image content.', parameters: {}, + execute: () => Promise.resolve(Array.from({ length: count }, () => ({ + type: 'image' as const, + attachment: { + attachmentId: 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' as never, + mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1, + }, + }))), + })) + runtime.behavior = async (request) => { + await request.bindings[0]!.functions[name]!({}) + return { output: [] } + } + + const result = await runCode(ctx, 'program') + expect(result.isError).toBe(true) + expect(result.content).toEqual([{ type: 'text', text: expect.stringContaining(message) as string }]) + }) + + it('returns CUA text observations as program values', async () => { + const { ctx, runtime } = await setup({ mode: 'ptc' }) + onTestFinished(() => ctx.fiber.dispose()) + ctx.tools.register(defineContentToolFixture({ + name: 'cua', description: 'Return a text observation.', parameters: {}, + execute: () => Promise.resolve([{ type: 'text', text: 'Window selected' }]), + })) + runtime.behavior = async request => printed(await request.bindings[0]!.functions.cua!({})) + + const result = await runCode(ctx, 'program') + expect(result.isError).toBe(false) + expect(result.content).toEqual([{ type: 'text', text: JSON.stringify([{ type: 'text', text: 'Window selected' }], null, 2) }]) + }) + it('does not defer images removed by a nested post-execute decision', async () => { for (const decision of ['block', 'replace'] as const) { const { ctx, runtime } = await setup({ mode: 'ptc' }) diff --git a/packages/core/tool-executor-ptc/tests/ts-types.spec.ts b/packages/core/tool-executor-ptc/tests/ts-types.spec.ts index 0cd107b90a..9873fe9503 100644 --- a/packages/core/tool-executor-ptc/tests/ts-types.spec.ts +++ b/packages/core/tool-executor-ptc/tests/ts-types.spec.ts @@ -153,6 +153,16 @@ describe('renderToolsSdk', () => { expect(text).toContain('lossless JSON') }) + it('declares Image results for read_image and image-or-value results for cua', () => { + const tools: ToolSdkSchema[] = ['read_image', 'cua'].map(name => ({ + name, description: 'Image-producing tool.', parameters: {}, output: { type: 'string' }, + })) + const text = renderToolsSdk(tools) + const outputs = text.slice(text.indexOf('interface ToolOutputMap'), text.indexOf('type ToolName')) + expect(outputs).toContain('read_image: Image;') + expect(outputs).toContain('cua: Image | string;') + }) + it('lists the program before its description', () => { const text = renderToolsSdk([bash]) expect(text).toContain('`code`') diff --git a/packages/document/tool-office/tests/tool-office.spec.ts b/packages/document/tool-office/tests/tool-office.spec.ts index 7e8b6f802f..81be68d2e7 100644 --- a/packages/document/tool-office/tests/tool-office.spec.ts +++ b/packages/document/tool-office/tests/tool-office.spec.ts @@ -648,12 +648,12 @@ describe('arguments and paths', () => { }) describe('PTC dispatch', () => { - it('forwards nested render images and applies execution policy to the nested call', async () => { + it('records nested render images and applies execution policy to the nested call', async () => { const { ctx } = await setup({ ptc: true }) const runtime = ctx.ptcRuntime as FakeRuntime runtime.behavior = async (request) => { - const value = await request.bindings[0]!.functions.office_render!({ input: 'deck.pptx' }) - return { output: [], value } + await request.bindings[0]!.functions.office_render!({ input: 'deck.pptx' }) + return { output: [] } } const agent = { ...agentOn('vision'), ctx } await loadOffice(ctx, agent) @@ -663,9 +663,12 @@ describe('PTC dispatch', () => { signal, callId: ToolCallId(`run-${++calls}`), name: RUN_CODE_NAME, arguments: args, agent: agent as never, toolSet, }) - const result = await run({ code: 'return await tools.office_render({ input: "deck.pptx" })', description: 'Render' }) + const result = await run({ code: 'await tools.office_render({ input: "deck.pptx" })', description: 'Render' }) expect(result.isError).toBe(false) - expect(result.additionalContexts?.[0]?.content.some(block => block.type === 'image')).toBe(true) + const dispatch = agent.session.snapshotEvents().find(event => event.type === 'tool/ptc-dispatch') + expect(dispatch?.data.content.some(block => block.type === 'image')).toBe(true) + expect(result.content.some(block => block.type === 'image')).toBe(false) + expect(result.additionalContexts).toBeUndefined() const denied: string[] = [] ctx.on('tools/pre-execute', (exec, next) => { @@ -673,7 +676,7 @@ describe('PTC dispatch', () => { denied.push(exec.name) return Promise.resolve({ kind: 'deny' as const, reason: 'office denied by policy' }) }, { prepend: true }) - const refused = await run({ code: 'return await tools.office_render({ input: "deck.pptx" })', description: 'Render' }) + const refused = await run({ code: 'await tools.office_render({ input: "deck.pptx" })', description: 'Render' }) expect(denied).toEqual(['office_render']) expect(text(refused)).toContain('office denied by policy') }) diff --git a/packages/fs/tool-fs/tests/read-image.spec.ts b/packages/fs/tool-fs/tests/read-image.spec.ts index 2d2eae7f3d..ab836e41dc 100644 --- a/packages/fs/tool-fs/tests/read-image.spec.ts +++ b/packages/fs/tool-fs/tests/read-image.spec.ts @@ -22,6 +22,7 @@ import { ToolCallId, LlmAdapter, LlmRuntime } from '@deepseek-ai/dsh-llm' import type { GenerateOptions, LlmModelInfo, LlmResolvedModelInfo, Message, StreamChunk } from '@deepseek-ai/dsh-llm' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' import ToolRuntime, { RUN_CODE_NAME } from '@deepseek-ai/dsh-tools' +import type { PostToolDecision } from '@deepseek-ai/dsh-tools' import LocalFileSystem from '@deepseek-ai/dsh-fs-local' import * as FsPolicy from '@deepseek-ai/dsh-fs-observation-policy' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' @@ -324,6 +325,46 @@ describe('read_image happy path', () => { ]) }) + it.each([ + ['around', 'block'], ['around', 'replace'], ['around', 'recover'], + ['post', 'block'], ['post', 'replace'], + ] as const)('honors an outer %s %s decision after a program inspects an image', async (stage, decision) => { + await writeFile(join(dir, 'red.png'), PNG_1X1) + const ctx = await setup({ toolMode: 'ptc' }) + const runtime = ctx.ptcRuntime as FakeRuntime + runtime.behavior = async (request) => { + const image = await request.bindings[0]!.functions.read_image!({ file_path: 'red.png' }) + if (!(image instanceof PtcImage)) throw new Error('read_image must return an Image') + return { + output: [{ type: 'image', attachment: image.attachment }], + ...decision === 'recover' ? { error: { kind: 'exception' as const, message: 'program stopped' } } : {}, + } + } + if (stage === 'around') { + ctx.on('tools/execute', async (exec, next) => { + const result = await next() + if (exec.name !== RUN_CODE_NAME) return result + return decision === 'block' + ? { isError: true, error: { message: 'image blocked' }, content: [{ type: 'text', text: 'image blocked' }] } + : { isError: false, value: { logs: ['image replaced'] }, content: [{ type: 'text', text: 'image replaced' }] } + }) + } else { + ctx.on('tools/post-execute', (exec, _result, next): Promise => { + if (exec.name !== RUN_CODE_NAME) return next() + return Promise.resolve(decision === 'block' + ? { kind: 'block', feedback: [{ type: 'text', text: 'image blocked' }] } + : { kind: 'accept', value: { logs: ['image replaced'] } }) + }) + } + + const result = await call(ctx, RUN_CODE_NAME, { + code: 'inspect(await tools.read_image({ file_path: "red.png" }))', description: 'Inspect an image', + }, { ...agentOn('vision-model'), ctx }) + + expect(result.isError).toBe(decision === 'block') + expect(result.content).toEqual([{ type: 'text', text: decision === 'block' ? 'image blocked' : 'image replaced' }]) + }) + it.each(['exception', 'timeout'] as const)('preserves inspected text and image order after an %s', async (kind) => { await writeFile(join(dir, 'red.png'), PNG_1X1) const ctx = await setup({ toolMode: 'ptc' }) diff --git a/packages/skill/skill-office/tests/render-tools.e2e.ts b/packages/skill/skill-office/tests/render-tools.e2e.ts index c6f085f40e..9091c5e451 100644 --- a/packages/skill/skill-office/tests/render-tools.e2e.ts +++ b/packages/skill/skill-office/tests/render-tools.e2e.ts @@ -63,12 +63,11 @@ it.skipIf(!process.env.DEEPSEEK_API_KEY || !runtime).each([{ format: 'docx', mod }) const renders = officeCalls(events).filter(call => call.name === 'office_render') expect(renders.length, JSON.stringify(calls)).toBeGreaterThan(0) - const renderCallIds = new Set(renders.map(call => call.callId)) - const images = events.some(event => mode === 'native' - ? event.type === 'tool/result' && renderCallIds.has(event.data.message.toolCallId) - && !event.data.message.isError && event.data.message.content.some(block => block.type === 'image') - : event.type === 'user/message' && event.data.source.kind === 'ptc-mode' - && event.data.content.some(block => block.type === 'image')) + const imageCallIds = new Set(mode === 'native' ? renders.map(call => call.callId) + : events.flatMap(event => event.type === 'tool/call' && event.data.name === 'run_code' ? [event.data.callId] : [])) + const images = events.some(event => event.type === 'tool/result' + && imageCallIds.has(event.data.message.toolCallId) && !event.data.message.isError + && event.data.message.content.some(block => block.type === 'image')) expect(images, JSON.stringify(calls)).toBe(true) }, }) diff --git a/packages/skill/skill-office/tests/xlsx-validation.e2e.ts b/packages/skill/skill-office/tests/xlsx-validation.e2e.ts index 1befea8162..7aa581b2d4 100644 --- a/packages/skill/skill-office/tests/xlsx-validation.e2e.ts +++ b/packages/skill/skill-office/tests/xlsx-validation.e2e.ts @@ -138,10 +138,8 @@ else: `], { cwd }) expect(verified.exitCode).toBe(0) if (kind === 'formula') expect(officeCalls(events).some(call => call.name === 'office_recalculate')).toBe(true) - const imageBlocks = events.flatMap(event => event.type === 'tool/result' - ? event.data.message.content.filter(block => block.type === 'image') - : event.type === 'user/message' && event.data.source.kind === 'ptc-mode' - ? event.data.content.filter(block => block.type === 'image') : []) + const imageBlocks = events.flatMap(event => event.type === 'tool/result' && !event.data.message.isError + ? event.data.message.content.filter(block => block.type === 'image') : []) for (const block of imageBlocks) { const stored = await readImageFile(join(imageRoot, 'attachments/v1'), block.attachment) const pixels = await execa(python!, ['-c', previewPixels], { input: Buffer.from(stored.data) }) @@ -172,8 +170,9 @@ else: const visibleImage = mode === 'native' ? results.some(result => !result.isError && reads.some(call => call.callId === result.toolCallId) && result.content.some(block => block.type === 'image')) - : events.some(event => event.type === 'user/message' && event.data.source.kind === 'ptc-mode' - && event.data.content.some(block => block.type === 'image')) + : results.some(result => !result.isError + && calls.some(call => call.name === 'run_code' && call.callId === result.toolCallId) + && result.content.some(block => block.type === 'image')) expect(visibleImage).toBe(true) } if (!previewHasContent) { diff --git a/packages/spill/spill-policy/tests/multimodal-recovery.spec.ts b/packages/spill/spill-policy/tests/multimodal-recovery.spec.ts index 96026b1ab7..d38dc3eeb6 100644 --- a/packages/spill/spill-policy/tests/multimodal-recovery.spec.ts +++ b/packages/spill/spill-policy/tests/multimodal-recovery.spec.ts @@ -198,18 +198,26 @@ describe('multimodal recovery through real providers', () => { ? 'await Promise.all([tools.ends({}), tools.middle({}), tools.small({})])' : '[await tools.ends({}), await tools.middle({}), await tools.small({})]' const result = await execute('run_code', { - code: `const [ends, middle, small] = ${calls}; inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length }));`, + code: `const [ends, middle, small] = ${calls}; +inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length })); +const fs = await import('node:fs/promises'); +for (const [index, image] of [ends.content[1], ends.content[3]].entries()) { + const path = 'recovered-' + index + '.png'; + await fs.writeFile(path, Buffer.from(image.data, 'base64')); + inspect(await tools.read_image({ file_path: path })); +}`, description: 'Inspect independent image result budgets', }) expect(result.isError).toBe(false) expect(result.value).toMatchObject({ logs: [JSON.stringify({ ends: LONG.length, middle: LONG.length, small: SMALL.length })] }) const dispatches = session.snapshotEvents().filter(event => event.type === 'tool/ptc-dispatch').map(event => event.data) - expect(dispatches).toHaveLength(3) + expect(dispatches).toHaveLength(5) for (const dispatch of dispatches) expect(cost(dispatch.content)).toBeLessThanOrEqual(CAP) expect(dispatches.find(event => event.name === 'ends')?.content.filter(block => block.type === 'image')).toHaveLength(2) expect(textOf(dispatches.find(event => event.name === 'middle')!.content)).toContain('Omitted 1 images.') expect(dispatches.find(event => event.name === 'small')?.content).toEqual([text(SMALL)]) - expect(result.additionalContexts).toHaveLength(2) + expect(imagesOf(result.content)).toHaveLength(2) + expect(imagesOf(result.additionalContexts?.flatMap(context => context.content) ?? [])).toEqual([]) expect(await requestImages(result.content, result.additionalContexts)).toEqual(['red', 'blue']) }) })