test(ptc): cover image output policies and explicit recovery
This commit is contained in:
1 parent
ac568aa2d2
commit
9f8fd9d800
8 files changed
+134
-25
No files matched your search
@@ -679,7 +679,7 @@ export function createRunCodeTool(
|
||||
if (runtime.language === 'typescript' && (name === 'read_image' || name === 'cua')) {
|
||||
const images = outcome.content.filter(block => block.type === 'image')
|
||||
const first = images[0]
|
||||
if (images.length === 1 && first !== undefined) return programImage(first.attachment)
|
||||
if (first !== undefined && images.length === 1) return programImage(first.attachment)
|
||||
if (name === 'read_image' || images.length > 1) throw new Error('Image-producing tools must return exactly one image')
|
||||
}
|
||||
return outcome.value
|
||||
|
||||
@@ -1651,8 +1651,8 @@ describe('the run_code dispatch bridge', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps image-bearing sub-call results out of the conversation until inspected', async () => {
|
||||
const { ctx, runtime } = await setup({ mode: 'ptc' })
|
||||
it.each(['typescript', 'python'])('records image-bearing sub-call results for a %s runtime', async (language) => {
|
||||
const { ctx, runtime } = await setup({ mode: 'ptc', runtime: { language } })
|
||||
ctx.tools.register(defineContentToolFixture({
|
||||
name: 'image_result',
|
||||
description: 'Return one durable image.',
|
||||
@@ -1675,10 +1675,59 @@ describe('the run_code dispatch bridge', () => {
|
||||
|
||||
const result = await runCode(ctx, 'program')
|
||||
|
||||
expect(result.additionalContexts).toBeUndefined()
|
||||
if (language === 'typescript') expect(result.additionalContexts).toBeUndefined()
|
||||
else expect(result.additionalContexts).toMatchObject([{
|
||||
source: { kind: 'ptc-mode' },
|
||||
content: [
|
||||
{ type: 'text', text: 'image result' },
|
||||
{ type: 'image', attachment: { attachmentId: 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' } },
|
||||
],
|
||||
}])
|
||||
expect(result.content).toEqual([{ type: 'text', text: 'done' }])
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ name: 'read_image', count: 0, message: 'Image-producing tools must return exactly one image' },
|
||||
{ name: 'read_image', count: 2, message: 'Image-producing tools must return exactly one image' },
|
||||
{ name: 'cua', count: 2, message: 'Image-producing tools must return exactly one image' },
|
||||
{ name: 'read_image', count: 1, message: 'Image storage must expose a readable path in the program filesystem' },
|
||||
])('rejects $name with $count images when its program image is unavailable', async ({ name, count, message }) => {
|
||||
const { ctx, runtime } = await setup({ mode: 'ptc' })
|
||||
onTestFinished(() => ctx.fiber.dispose())
|
||||
ctx.tools.register(defineContentToolFixture({
|
||||
name, description: 'Return image content.', parameters: {},
|
||||
execute: () => Promise.resolve(Array.from({ length: count }, () => ({
|
||||
type: 'image' as const,
|
||||
attachment: {
|
||||
attachmentId: 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' as never,
|
||||
mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1,
|
||||
},
|
||||
}))),
|
||||
}))
|
||||
runtime.behavior = async (request) => {
|
||||
await request.bindings[0]!.functions[name]!({})
|
||||
return { output: [] }
|
||||
}
|
||||
|
||||
const result = await runCode(ctx, 'program')
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.content).toEqual([{ type: 'text', text: expect.stringContaining(message) as string }])
|
||||
})
|
||||
|
||||
it('returns CUA text observations as program values', async () => {
|
||||
const { ctx, runtime } = await setup({ mode: 'ptc' })
|
||||
onTestFinished(() => ctx.fiber.dispose())
|
||||
ctx.tools.register(defineContentToolFixture({
|
||||
name: 'cua', description: 'Return a text observation.', parameters: {},
|
||||
execute: () => Promise.resolve([{ type: 'text', text: 'Window selected' }]),
|
||||
}))
|
||||
runtime.behavior = async request => printed(await request.bindings[0]!.functions.cua!({}))
|
||||
|
||||
const result = await runCode(ctx, 'program')
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.content).toEqual([{ type: 'text', text: JSON.stringify([{ type: 'text', text: 'Window selected' }], null, 2) }])
|
||||
})
|
||||
|
||||
it('does not defer images removed by a nested post-execute decision', async () => {
|
||||
for (const decision of ['block', 'replace'] as const) {
|
||||
const { ctx, runtime } = await setup({ mode: 'ptc' })
|
||||
|
||||
@@ -153,6 +153,16 @@ describe('renderToolsSdk', () => {
|
||||
expect(text).toContain('lossless JSON')
|
||||
})
|
||||
|
||||
it('declares Image results for read_image and image-or-value results for cua', () => {
|
||||
const tools: ToolSdkSchema[] = ['read_image', 'cua'].map(name => ({
|
||||
name, description: 'Image-producing tool.', parameters: {}, output: { type: 'string' },
|
||||
}))
|
||||
const text = renderToolsSdk(tools)
|
||||
const outputs = text.slice(text.indexOf('interface ToolOutputMap'), text.indexOf('type ToolName'))
|
||||
expect(outputs).toContain('read_image: Image;')
|
||||
expect(outputs).toContain('cua: Image | string;')
|
||||
})
|
||||
|
||||
it('lists the program before its description', () => {
|
||||
const text = renderToolsSdk([bash])
|
||||
expect(text).toContain('`code`')
|
||||
|
||||
@@ -648,12 +648,12 @@ describe('arguments and paths', () => {
|
||||
})
|
||||
|
||||
describe('PTC dispatch', () => {
|
||||
it('forwards nested render images and applies execution policy to the nested call', async () => {
|
||||
it('records nested render images and applies execution policy to the nested call', async () => {
|
||||
const { ctx } = await setup({ ptc: true })
|
||||
const runtime = ctx.ptcRuntime as FakeRuntime
|
||||
runtime.behavior = async (request) => {
|
||||
const value = await request.bindings[0]!.functions.office_render!({ input: 'deck.pptx' })
|
||||
return { output: [], value }
|
||||
await request.bindings[0]!.functions.office_render!({ input: 'deck.pptx' })
|
||||
return { output: [] }
|
||||
}
|
||||
const agent = { ...agentOn('vision'), ctx }
|
||||
await loadOffice(ctx, agent)
|
||||
@@ -663,9 +663,12 @@ describe('PTC dispatch', () => {
|
||||
signal, callId: ToolCallId(`run-${++calls}`), name: RUN_CODE_NAME, arguments: args, agent: agent as never,
|
||||
toolSet,
|
||||
})
|
||||
const result = await run({ code: 'return await tools.office_render({ input: "deck.pptx" })', description: 'Render' })
|
||||
const result = await run({ code: 'await tools.office_render({ input: "deck.pptx" })', description: 'Render' })
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.additionalContexts?.[0]?.content.some(block => block.type === 'image')).toBe(true)
|
||||
const dispatch = agent.session.snapshotEvents().find(event => event.type === 'tool/ptc-dispatch')
|
||||
expect(dispatch?.data.content.some(block => block.type === 'image')).toBe(true)
|
||||
expect(result.content.some(block => block.type === 'image')).toBe(false)
|
||||
expect(result.additionalContexts).toBeUndefined()
|
||||
|
||||
const denied: string[] = []
|
||||
ctx.on('tools/pre-execute', (exec, next) => {
|
||||
@@ -673,7 +676,7 @@ describe('PTC dispatch', () => {
|
||||
denied.push(exec.name)
|
||||
return Promise.resolve({ kind: 'deny' as const, reason: 'office denied by policy' })
|
||||
}, { prepend: true })
|
||||
const refused = await run({ code: 'return await tools.office_render({ input: "deck.pptx" })', description: 'Render' })
|
||||
const refused = await run({ code: 'await tools.office_render({ input: "deck.pptx" })', description: 'Render' })
|
||||
expect(denied).toEqual(['office_render'])
|
||||
expect(text(refused)).toContain('office denied by policy')
|
||||
})
|
||||
|
||||
@@ -22,6 +22,7 @@ import { ToolCallId, LlmAdapter, LlmRuntime } from '@deepseek-ai/dsh-llm'
|
||||
import type { GenerateOptions, LlmModelInfo, LlmResolvedModelInfo, Message, StreamChunk } from '@deepseek-ai/dsh-llm'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRuntime, { RUN_CODE_NAME } from '@deepseek-ai/dsh-tools'
|
||||
import type { PostToolDecision } from '@deepseek-ai/dsh-tools'
|
||||
import LocalFileSystem from '@deepseek-ai/dsh-fs-local'
|
||||
import * as FsPolicy from '@deepseek-ai/dsh-fs-observation-policy'
|
||||
import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local'
|
||||
@@ -324,6 +325,46 @@ describe('read_image happy path', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it.each([
|
||||
['around', 'block'], ['around', 'replace'], ['around', 'recover'],
|
||||
['post', 'block'], ['post', 'replace'],
|
||||
] as const)('honors an outer %s %s decision after a program inspects an image', async (stage, decision) => {
|
||||
await writeFile(join(dir, 'red.png'), PNG_1X1)
|
||||
const ctx = await setup({ toolMode: 'ptc' })
|
||||
const runtime = ctx.ptcRuntime as FakeRuntime
|
||||
runtime.behavior = async (request) => {
|
||||
const image = await request.bindings[0]!.functions.read_image!({ file_path: 'red.png' })
|
||||
if (!(image instanceof PtcImage)) throw new Error('read_image must return an Image')
|
||||
return {
|
||||
output: [{ type: 'image', attachment: image.attachment }],
|
||||
...decision === 'recover' ? { error: { kind: 'exception' as const, message: 'program stopped' } } : {},
|
||||
}
|
||||
}
|
||||
if (stage === 'around') {
|
||||
ctx.on('tools/execute', async (exec, next) => {
|
||||
const result = await next()
|
||||
if (exec.name !== RUN_CODE_NAME) return result
|
||||
return decision === 'block'
|
||||
? { isError: true, error: { message: 'image blocked' }, content: [{ type: 'text', text: 'image blocked' }] }
|
||||
: { isError: false, value: { logs: ['image replaced'] }, content: [{ type: 'text', text: 'image replaced' }] }
|
||||
})
|
||||
} else {
|
||||
ctx.on('tools/post-execute', (exec, _result, next): Promise<PostToolDecision> => {
|
||||
if (exec.name !== RUN_CODE_NAME) return next()
|
||||
return Promise.resolve(decision === 'block'
|
||||
? { kind: 'block', feedback: [{ type: 'text', text: 'image blocked' }] }
|
||||
: { kind: 'accept', value: { logs: ['image replaced'] } })
|
||||
})
|
||||
}
|
||||
|
||||
const result = await call(ctx, RUN_CODE_NAME, {
|
||||
code: 'inspect(await tools.read_image({ file_path: "red.png" }))', description: 'Inspect an image',
|
||||
}, { ...agentOn('vision-model'), ctx })
|
||||
|
||||
expect(result.isError).toBe(decision === 'block')
|
||||
expect(result.content).toEqual([{ type: 'text', text: decision === 'block' ? 'image blocked' : 'image replaced' }])
|
||||
})
|
||||
|
||||
it.each(['exception', 'timeout'] as const)('preserves inspected text and image order after an %s', async (kind) => {
|
||||
await writeFile(join(dir, 'red.png'), PNG_1X1)
|
||||
const ctx = await setup({ toolMode: 'ptc' })
|
||||
|
||||
@@ -63,12 +63,11 @@ it.skipIf(!process.env.DEEPSEEK_API_KEY || !runtime).each([{ format: 'docx', mod
|
||||
})
|
||||
const renders = officeCalls(events).filter(call => call.name === 'office_render')
|
||||
expect(renders.length, JSON.stringify(calls)).toBeGreaterThan(0)
|
||||
const renderCallIds = new Set(renders.map(call => call.callId))
|
||||
const images = events.some(event => mode === 'native'
|
||||
? event.type === 'tool/result' && renderCallIds.has(event.data.message.toolCallId)
|
||||
&& !event.data.message.isError && event.data.message.content.some(block => block.type === 'image')
|
||||
: event.type === 'user/message' && event.data.source.kind === 'ptc-mode'
|
||||
&& event.data.content.some(block => block.type === 'image'))
|
||||
const imageCallIds = new Set(mode === 'native' ? renders.map(call => call.callId)
|
||||
: events.flatMap(event => event.type === 'tool/call' && event.data.name === 'run_code' ? [event.data.callId] : []))
|
||||
const images = events.some(event => event.type === 'tool/result'
|
||||
&& imageCallIds.has(event.data.message.toolCallId) && !event.data.message.isError
|
||||
&& event.data.message.content.some(block => block.type === 'image'))
|
||||
expect(images, JSON.stringify(calls)).toBe(true)
|
||||
},
|
||||
})
|
||||
|
||||
@@ -138,10 +138,8 @@ else:
|
||||
`], { cwd })
|
||||
expect(verified.exitCode).toBe(0)
|
||||
if (kind === 'formula') expect(officeCalls(events).some(call => call.name === 'office_recalculate')).toBe(true)
|
||||
const imageBlocks = events.flatMap(event => event.type === 'tool/result'
|
||||
? event.data.message.content.filter(block => block.type === 'image')
|
||||
: event.type === 'user/message' && event.data.source.kind === 'ptc-mode'
|
||||
? event.data.content.filter(block => block.type === 'image') : [])
|
||||
const imageBlocks = events.flatMap(event => event.type === 'tool/result' && !event.data.message.isError
|
||||
? event.data.message.content.filter(block => block.type === 'image') : [])
|
||||
for (const block of imageBlocks) {
|
||||
const stored = await readImageFile(join(imageRoot, 'attachments/v1'), block.attachment)
|
||||
const pixels = await execa(python!, ['-c', previewPixels], { input: Buffer.from(stored.data) })
|
||||
@@ -172,8 +170,9 @@ else:
|
||||
const visibleImage = mode === 'native'
|
||||
? results.some(result => !result.isError && reads.some(call => call.callId === result.toolCallId)
|
||||
&& result.content.some(block => block.type === 'image'))
|
||||
: events.some(event => event.type === 'user/message' && event.data.source.kind === 'ptc-mode'
|
||||
&& event.data.content.some(block => block.type === 'image'))
|
||||
: results.some(result => !result.isError
|
||||
&& calls.some(call => call.name === 'run_code' && call.callId === result.toolCallId)
|
||||
&& result.content.some(block => block.type === 'image'))
|
||||
expect(visibleImage).toBe(true)
|
||||
}
|
||||
if (!previewHasContent) {
|
||||
|
||||
@@ -198,18 +198,26 @@ describe('multimodal recovery through real providers', () => {
|
||||
? 'await Promise.all([tools.ends({}), tools.middle({}), tools.small({})])'
|
||||
: '[await tools.ends({}), await tools.middle({}), await tools.small({})]'
|
||||
const result = await execute('run_code', {
|
||||
code: `const [ends, middle, small] = ${calls}; inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length }));`,
|
||||
code: `const [ends, middle, small] = ${calls};
|
||||
inspect(JSON.stringify({ ends: ends.content[2].text.length, middle: middle.content[0].text.length, small: small.content[0].text.length }));
|
||||
const fs = await import('node:fs/promises');
|
||||
for (const [index, image] of [ends.content[1], ends.content[3]].entries()) {
|
||||
const path = 'recovered-' + index + '.png';
|
||||
await fs.writeFile(path, Buffer.from(image.data, 'base64'));
|
||||
inspect(await tools.read_image({ file_path: path }));
|
||||
}`,
|
||||
description: 'Inspect independent image result budgets',
|
||||
})
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.value).toMatchObject({ logs: [JSON.stringify({ ends: LONG.length, middle: LONG.length, small: SMALL.length })] })
|
||||
const dispatches = session.snapshotEvents().filter(event => event.type === 'tool/ptc-dispatch').map(event => event.data)
|
||||
expect(dispatches).toHaveLength(3)
|
||||
expect(dispatches).toHaveLength(5)
|
||||
for (const dispatch of dispatches) expect(cost(dispatch.content)).toBeLessThanOrEqual(CAP)
|
||||
expect(dispatches.find(event => event.name === 'ends')?.content.filter(block => block.type === 'image')).toHaveLength(2)
|
||||
expect(textOf(dispatches.find(event => event.name === 'middle')!.content)).toContain('Omitted 1 images.')
|
||||
expect(dispatches.find(event => event.name === 'small')?.content).toEqual([text(SMALL)])
|
||||
expect(result.additionalContexts).toHaveLength(2)
|
||||
expect(imagesOf(result.content)).toHaveLength(2)
|
||||
expect(imagesOf(result.additionalContexts?.flatMap(context => context.content) ?? [])).toEqual([])
|
||||
expect(await requestImages(result.content, result.additionalContexts)).toEqual(['red', 'blue'])
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user