test(office): include PTC context images in visual checks

This commit is contained in:
_Kerman committed 2026-11-09 22:06:24 +08:00
1 parent 5bcee95efe
commit 9b3119f1f9
4 files changed
+46 -24

No files matched your search

@@ -1,5 +1,6 @@
/** Shared model configuration and context assertions for real Office workflows. */
import { expect } from 'vitest'
import type { ImageBlock } from '@deepseek-ai/dsh-llm'
import { Session, SessionId, type SessionEvent } from '@deepseek-ai/dsh-session'
/** @returns Model provider patches for the configured development or official API. */
@@ -52,6 +53,19 @@ export function officeCalls(events: readonly SessionEvent[]): OfficeCall[] {
})
}
/**
* Collect images displayed during an Office task.
* @param events - Recorded task events.
* @returns Images in successful tool results and PTC additional context, excluding private dispatch output.
*/
export function officeImages(events: readonly SessionEvent[]): ImageBlock[] {
return events.flatMap((event) => {
const content = event.type === 'tool/result' && !event.data.message.isError ? event.data.message.content
: event.type === 'user/message' && event.data.source.kind === 'ptc-mode' ? event.data.content : []
return content.filter(block => block.type === 'image')
})
}
/**
* Reject recorded Office render calls and the "Render the saved deliverable" instruction.
* @param events - Recorded task events, including nested skill and shell results.
@@ -1,8 +1,34 @@
/** Render detection uses recorded tool names, not JSON spelling or quoted skill instructions. */
import { createUserMessage, ToolCallId } from '@deepseek-ai/dsh-llm'
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
import { createToolResultMessage, createUserMessage, ToolCallId } from '@deepseek-ai/dsh-llm'
import type { ImageBlock } from '@deepseek-ai/dsh-llm'
import { Session, SessionId } from '@deepseek-ai/dsh-session'
import { expect, it } from 'vitest'
import { expectNoRenderContext, officeCalls } from './fixtures/real-office.ts'
import { expectNoRenderContext, officeCalls, officeImages } from './fixtures/real-office.ts'
it('collects displayed images from native results, inspect output and PTC additional context', () => {
const session = Session.create(SessionId('displayed-images'))
const image: ImageBlock = { type: 'image', attachment: {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`), mediaType: 'image/png', bytes: 64, width: 8, height: 4,
} }
session.append('tool/ptc-dispatch', {
rootCallId: ToolCallId('program'), parentCallId: ToolCallId('program'), subCallId: ToolCallId('program:1'),
name: 'office_render', arguments: {}, isError: false, content: [image],
})
session.append('user/message', createUserMessage({ content: [image], source: { kind: 'user' } }), { surfaceOp: 'append' })
session.append('tool/result', { turn: 1, step: 1,
message: createToolResultMessage({ callId: ToolCallId('failed'), content: [image], isError: true }),
}, { surfaceOp: 'append' })
expect(officeImages(session.snapshotEvents())).toEqual([])
for (const callId of ['render', 'program']) {
session.append('tool/result', { turn: 1, step: 1,
message: createToolResultMessage({ callId: ToolCallId(callId), content: [image], isError: false }),
}, { surfaceOp: 'append' })
}
session.append('user/message', createUserMessage({ content: [image], source: { kind: 'ptc-mode' } }), { surfaceOp: 'append' })
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'image omitted' }], source: { kind: 'ptc-mode' } }), { surfaceOp: 'append' })
expect(officeImages(session.snapshotEvents())).toEqual([image, image, image])
})
it.each([undefined, 2])('rejects a native render with JSON indentation %s', (indent) => {
const session = Session.create(SessionId('native-render'))
@@ -6,7 +6,7 @@ import { execa } from 'execa'
import { expect, it } from 'vitest'
import { runLoaderSmoke } from '@deepseek-ai/dsh-loader-smoke'
import { parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
import { expectNoRenderContext, officeCalls, officeModelPatches, officeToolPatches } from './fixtures/real-office.ts'
import { expectNoRenderContext, officeCalls, officeImages, officeModelPatches, officeToolPatches } from './fixtures/real-office.ts'
const repo = fileURLToPath(new URL('../../../../', import.meta.url))
const runtime = process.env.DSH_PRIMARY_RUNTIME
@@ -63,12 +63,7 @@ it.skipIf(!process.env.DEEPSEEK_API_KEY || !runtime).each([{ format: 'docx', mod
})
const renders = officeCalls(events).filter(call => call.name === 'office_render')
expect(renders.length, JSON.stringify(calls)).toBeGreaterThan(0)
const imageCallIds = new Set(mode === 'native' ? renders.map(call => call.callId)
: events.flatMap(event => event.type === 'tool/call' && event.data.name === 'run_code' ? [event.data.callId] : []))
const images = events.some(event => event.type === 'tool/result'
&& imageCallIds.has(event.data.message.toolCallId) && !event.data.message.isError
&& event.data.message.content.some(block => block.type === 'image'))
expect(images, JSON.stringify(calls)).toBe(true)
expect(officeImages(events).length, JSON.stringify(calls)).toBeGreaterThan(0)
},
})
})
@@ -9,7 +9,7 @@ import { runLoaderSmoke } from '@deepseek-ai/dsh-loader-smoke'
import { readImageFile } from '@deepseek-ai/dsh-attachment-local'
import { parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
import type { SessionEvent } from '@deepseek-ai/dsh-session'
import { expectNoRenderContext, officeCalls, officeModelPatches, officeToolPatches } from './fixtures/real-office.ts'
import { expectNoRenderContext, officeCalls, officeImages, officeModelPatches, officeToolPatches } from './fixtures/real-office.ts'
const repo = fileURLToPath(new URL('../../../../', import.meta.url))
const assetRoot = fileURLToPath(new URL('../assets/', import.meta.url))
@@ -138,9 +138,7 @@ else:
`], { cwd })
expect(verified.exitCode).toBe(0)
if (kind === 'formula') expect(officeCalls(events).some(call => call.name === 'office_recalculate')).toBe(true)
const imageBlocks = events.flatMap(event => event.type === 'tool/result' && !event.data.message.isError
? event.data.message.content.filter(block => block.type === 'image') : [])
for (const block of imageBlocks) {
for (const block of officeImages(events)) {
const stored = await readImageFile(join(imageRoot, 'attachments/v1'), block.attachment)
const pixels = await execa(python!, ['-c', previewPixels], { input: Buffer.from(stored.data) })
previewHasContent ||= pixels.stdout === 'content'
@@ -164,17 +162,6 @@ else:
expect(office.some(call => call.name === 'office_render' && 'sheet' in call.args && 'range' in call.args), diagnostic).toBe(true)
expect(office.some(call => call.name === 'office_convert'), diagnostic).toBe(false)
const reads = calls.filter(call => call.name === 'office_render')
const results = events.flatMap(event => event.type === 'tool/result' ? [event.data.message] : [])
if (previewHasContent) {
expect(reads.length).toBeGreaterThan(0)
const visibleImage = mode === 'native'
? results.some(result => !result.isError && reads.some(call => call.callId === result.toolCallId)
&& result.content.some(block => block.type === 'image'))
: results.some(result => !result.isError
&& calls.some(call => call.name === 'run_code' && call.callId === result.toolCallId)
&& result.content.some(block => block.type === 'image'))
expect(visibleImage).toBe(true)
}
if (!previewHasContent) {
const answer = events.findLast(event => event.type === 'assistant/message')
if (answer?.type !== 'assistant/message') throw new Error('Agent did not finish with an answer')