test(office): include PTC context images in visual checks
This commit is contained in:
1 parent
5bcee95efe
commit
9b3119f1f9
4 files changed
+46
-24
No files matched your search
@@ -1,5 +1,6 @@
|
||||
/** Shared model configuration and context assertions for real Office workflows. */
|
||||
import { expect } from 'vitest'
|
||||
import type { ImageBlock } from '@deepseek-ai/dsh-llm'
|
||||
import { Session, SessionId, type SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
|
||||
/** @returns Model provider patches for the configured development or official API. */
|
||||
@@ -52,6 +53,19 @@ export function officeCalls(events: readonly SessionEvent[]): OfficeCall[] {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect images displayed during an Office task.
|
||||
* @param events - Recorded task events.
|
||||
* @returns Images in successful tool results and PTC additional context, excluding private dispatch output.
|
||||
*/
|
||||
export function officeImages(events: readonly SessionEvent[]): ImageBlock[] {
|
||||
return events.flatMap((event) => {
|
||||
const content = event.type === 'tool/result' && !event.data.message.isError ? event.data.message.content
|
||||
: event.type === 'user/message' && event.data.source.kind === 'ptc-mode' ? event.data.content : []
|
||||
return content.filter(block => block.type === 'image')
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject recorded Office render calls and the "Render the saved deliverable" instruction.
|
||||
* @param events - Recorded task events, including nested skill and shell results.
|
||||
|
||||
@@ -1,8 +1,34 @@
|
||||
/** Render detection uses recorded tool names, not JSON spelling or quoted skill instructions. */
|
||||
import { createUserMessage, ToolCallId } from '@deepseek-ai/dsh-llm'
|
||||
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
|
||||
import { createToolResultMessage, createUserMessage, ToolCallId } from '@deepseek-ai/dsh-llm'
|
||||
import type { ImageBlock } from '@deepseek-ai/dsh-llm'
|
||||
import { Session, SessionId } from '@deepseek-ai/dsh-session'
|
||||
import { expect, it } from 'vitest'
|
||||
import { expectNoRenderContext, officeCalls } from './fixtures/real-office.ts'
|
||||
import { expectNoRenderContext, officeCalls, officeImages } from './fixtures/real-office.ts'
|
||||
|
||||
it('collects displayed images from native results, inspect output and PTC additional context', () => {
|
||||
const session = Session.create(SessionId('displayed-images'))
|
||||
const image: ImageBlock = { type: 'image', attachment: {
|
||||
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`), mediaType: 'image/png', bytes: 64, width: 8, height: 4,
|
||||
} }
|
||||
session.append('tool/ptc-dispatch', {
|
||||
rootCallId: ToolCallId('program'), parentCallId: ToolCallId('program'), subCallId: ToolCallId('program:1'),
|
||||
name: 'office_render', arguments: {}, isError: false, content: [image],
|
||||
})
|
||||
session.append('user/message', createUserMessage({ content: [image], source: { kind: 'user' } }), { surfaceOp: 'append' })
|
||||
session.append('tool/result', { turn: 1, step: 1,
|
||||
message: createToolResultMessage({ callId: ToolCallId('failed'), content: [image], isError: true }),
|
||||
}, { surfaceOp: 'append' })
|
||||
expect(officeImages(session.snapshotEvents())).toEqual([])
|
||||
for (const callId of ['render', 'program']) {
|
||||
session.append('tool/result', { turn: 1, step: 1,
|
||||
message: createToolResultMessage({ callId: ToolCallId(callId), content: [image], isError: false }),
|
||||
}, { surfaceOp: 'append' })
|
||||
}
|
||||
session.append('user/message', createUserMessage({ content: [image], source: { kind: 'ptc-mode' } }), { surfaceOp: 'append' })
|
||||
session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'image omitted' }], source: { kind: 'ptc-mode' } }), { surfaceOp: 'append' })
|
||||
expect(officeImages(session.snapshotEvents())).toEqual([image, image, image])
|
||||
})
|
||||
|
||||
it.each([undefined, 2])('rejects a native render with JSON indentation %s', (indent) => {
|
||||
const session = Session.create(SessionId('native-render'))
|
||||
|
||||
@@ -6,7 +6,7 @@ import { execa } from 'execa'
|
||||
import { expect, it } from 'vitest'
|
||||
import { runLoaderSmoke } from '@deepseek-ai/dsh-loader-smoke'
|
||||
import { parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
|
||||
import { expectNoRenderContext, officeCalls, officeModelPatches, officeToolPatches } from './fixtures/real-office.ts'
|
||||
import { expectNoRenderContext, officeCalls, officeImages, officeModelPatches, officeToolPatches } from './fixtures/real-office.ts'
|
||||
|
||||
const repo = fileURLToPath(new URL('../../../../', import.meta.url))
|
||||
const runtime = process.env.DSH_PRIMARY_RUNTIME
|
||||
@@ -63,12 +63,7 @@ it.skipIf(!process.env.DEEPSEEK_API_KEY || !runtime).each([{ format: 'docx', mod
|
||||
})
|
||||
const renders = officeCalls(events).filter(call => call.name === 'office_render')
|
||||
expect(renders.length, JSON.stringify(calls)).toBeGreaterThan(0)
|
||||
const imageCallIds = new Set(mode === 'native' ? renders.map(call => call.callId)
|
||||
: events.flatMap(event => event.type === 'tool/call' && event.data.name === 'run_code' ? [event.data.callId] : []))
|
||||
const images = events.some(event => event.type === 'tool/result'
|
||||
&& imageCallIds.has(event.data.message.toolCallId) && !event.data.message.isError
|
||||
&& event.data.message.content.some(block => block.type === 'image'))
|
||||
expect(images, JSON.stringify(calls)).toBe(true)
|
||||
expect(officeImages(events).length, JSON.stringify(calls)).toBeGreaterThan(0)
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
@@ -9,7 +9,7 @@ import { runLoaderSmoke } from '@deepseek-ai/dsh-loader-smoke'
|
||||
import { readImageFile } from '@deepseek-ai/dsh-attachment-local'
|
||||
import { parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import { expectNoRenderContext, officeCalls, officeModelPatches, officeToolPatches } from './fixtures/real-office.ts'
|
||||
import { expectNoRenderContext, officeCalls, officeImages, officeModelPatches, officeToolPatches } from './fixtures/real-office.ts'
|
||||
|
||||
const repo = fileURLToPath(new URL('../../../../', import.meta.url))
|
||||
const assetRoot = fileURLToPath(new URL('../assets/', import.meta.url))
|
||||
@@ -138,9 +138,7 @@ else:
|
||||
`], { cwd })
|
||||
expect(verified.exitCode).toBe(0)
|
||||
if (kind === 'formula') expect(officeCalls(events).some(call => call.name === 'office_recalculate')).toBe(true)
|
||||
const imageBlocks = events.flatMap(event => event.type === 'tool/result' && !event.data.message.isError
|
||||
? event.data.message.content.filter(block => block.type === 'image') : [])
|
||||
for (const block of imageBlocks) {
|
||||
for (const block of officeImages(events)) {
|
||||
const stored = await readImageFile(join(imageRoot, 'attachments/v1'), block.attachment)
|
||||
const pixels = await execa(python!, ['-c', previewPixels], { input: Buffer.from(stored.data) })
|
||||
previewHasContent ||= pixels.stdout === 'content'
|
||||
@@ -164,17 +162,6 @@ else:
|
||||
expect(office.some(call => call.name === 'office_render' && 'sheet' in call.args && 'range' in call.args), diagnostic).toBe(true)
|
||||
expect(office.some(call => call.name === 'office_convert'), diagnostic).toBe(false)
|
||||
const reads = calls.filter(call => call.name === 'office_render')
|
||||
const results = events.flatMap(event => event.type === 'tool/result' ? [event.data.message] : [])
|
||||
if (previewHasContent) {
|
||||
expect(reads.length).toBeGreaterThan(0)
|
||||
const visibleImage = mode === 'native'
|
||||
? results.some(result => !result.isError && reads.some(call => call.callId === result.toolCallId)
|
||||
&& result.content.some(block => block.type === 'image'))
|
||||
: results.some(result => !result.isError
|
||||
&& calls.some(call => call.name === 'run_code' && call.callId === result.toolCallId)
|
||||
&& result.content.some(block => block.type === 'image'))
|
||||
expect(visibleImage).toBe(true)
|
||||
}
|
||||
if (!previewHasContent) {
|
||||
const answer = events.findLast(event => event.type === 'assistant/message')
|
||||
if (answer?.type !== 'assistant/message') throw new Error('Agent did not finish with an answer')
|
||||
|
||||
Reference in new issue
Block a user