mward4/deepseek-harness
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

test(cua): record foreground observations and align invocation assertions

Browse Source
This commit is contained in:
creatixchu committed 2026-09-24 16:54:33 +08:00
1 parent feba1df810
commit bc6ec2b1cd
3 files changed
+26 -16
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/experimental/cua/tests/large-image-ptc.spec.ts
+1 -1
View File
@@ -147,7 +147,7 @@ createInterface({ input: process.stdin }).on('line', line => {
const stop = context.on('agent/status', ({ agent, status }) => {
if (agent === handle.agent && status === 'idle') idle.resolve()
})
onTestFinished(stop)
onTestFinished(() => { stop() })
handle.agent.followup(createUserMessage({ source: { kind: 'user' }, content: [{ type: 'text', text: 'Inspect the generated screenshot.' }] }))
await idle.promise
stop()
packages/experimental/cua/tests/provider-lifecycle.spec.ts
+2 -2
View File
@@ -218,9 +218,9 @@ it('rejects malformed and unowned operations before acquiring a helper', async (
it('supplies configured resource bounds and clears registrations on unload', async () => {
const f = await setup()
await f.execute('getState', { appHandle: 'app-one' })
expect(native.request).toHaveBeenLastCalledWith('getState', { appHandle: 'app-one', maxElements: 64, maxDepth: 8 }, expect.any(AbortSignal))
expect(native.request).toHaveBeenLastCalledWith('getState', { appHandle: 'app-one', maxElements: 64, maxDepth: 8 }, expect.any(AbortSignal), ToolCallId('lifecycle-1'))
await f.execute('drag', { appHandle: 'app-one', windowId: 7, fromX: 1, fromY: 2, toX: 3, toY: 4 })
expect(native.request).toHaveBeenLastCalledWith('drag', expect.objectContaining({ durationMs: 200, steps: 10 }), expect.any(AbortSignal))
expect(native.request).toHaveBeenLastCalledWith('drag', expect.objectContaining({ durationMs: 200, steps: 10 }), expect.any(AbortSignal), ToolCallId('lifecycle-2'))
await f.provider.dispose()
expect(f.ctx.tools.schemas()).toEqual([])
expect(native.dispose).toHaveBeenCalledTimes(1)
snapshots/session/cua-native/session.v4.jsonl
+23 -13
View File
@@ -12,61 +12,71 @@
{"type":"request/header","data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"tools":"{{tools}}"},"reason":"initial"}}
{"type":"request/context","data":{"provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"}}
{"type":"session/title","data":{"title":"Inspect CUA capabilities, select the","messageSeqs":[8],"source":{"kind":"fallback"}}}
{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235068635,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235068635,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1790235068635,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235068635,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027555,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027555,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}},{"type":"chunk","time":1790240027555,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027555,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":1,"callId":"cua-native-1","name":"cua","arguments":"{\"method\":\"getCapabilities\",\"params\":{}}"}}
{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"platform\":\"macos\",\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"methods\":[\"getCapabilities\",\"listApps\",\"getApp\",\"listWindows\",\"getState\",\"click\",\"setValue\",\"typeText\",\"pressKey\",\"scroll\",\"drag\",\"screenshot\"],\"coordinateSpace\":\"window-points\",\"browser\":false}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"}
{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-1"},"toolCallId":"cua-native-1","content":[{"type":"text","text":"{\"platform\":\"macos\",\"sessionLocked\":false,\"permissions\":{\"accessibility\":true,\"screenRecording\":true},\"methods\":[\"getCapabilities\",\"listApps\",\"getApp\",\"activate\",\"listWindows\",\"getState\",\"click\",\"setValue\",\"typeText\",\"pressKey\",\"scroll\",\"drag\",\"screenshot\"],\"coordinateSpace\":\"window-points\",\"browser\":false}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":1}}
{"type":"step/start","data":{"turn":1,"step":2}}
{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235068899,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235068899,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}}},{"type":"chunk","time":1790235068899,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235068899,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:6}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027815,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027815,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}}},{"type":"chunk","time":1790240027815,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027815,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":2,"callId":"cua-native-2","name":"cua","arguments":"{\"method\":\"getApp\",\"params\":{\"selector\":\"test.fixture\"}}"}}
{"type":"tool/result","data":{"turn":1,"step":2,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-2"},"toolCallId":"cua-native-2","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"name\":\"Fixture\",\"bundleId\":\"test.fixture\"}"}],"isError":false,"id":"{{message:7}}"}},"sourceEventSeqs":[19],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":2}}
{"type":"step/start","data":{"turn":1,"step":3}}
{"type":"assistant/message","data":{"turn":1,"step":3,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:8}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235068913,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235068913,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1790235068913,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235068913,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":3,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:8}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027830,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027830,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1790240027830,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027830,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":3,"callId":"cua-native-3","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}
{"type":"tool/result","data":{"turn":1,"step":3,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-3"},"toolCallId":"cua-native-3","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-1\",\"tree\":\"0 AXButton Increment\\n1 AXTextField value: \\n2 AXStaticText Count: 0\",\"elements\":[{\"index\":0,\"role\":\"AXButton\",\"title\":\"Increment\",\"bounds\":{\"x\":60,\"y\":80,\"width\":100,\"height\":20}}],\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:9}}"}},"sourceEventSeqs":[24],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":3}}
{"type":"step/start","data":{"turn":1,"step":4}}
{"type":"assistant/message","data":{"turn":1,"step":4,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:10}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235068928,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235068928,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1790235068928,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235068928,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":4,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:10}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027845,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027845,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}}},{"type":"chunk","time":1790240027845,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027845,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":4,"callId":"cua-native-4","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"snapshotId\":\"fixture-snapshot-1\",\"elementIndex\":0}}"}}
{"type":"tool/result","data":{"turn":1,"step":4,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-4"},"toolCallId":"cua-native-4","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:11}}"}},"sourceEventSeqs":[29],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":4}}
{"type":"step/start","data":{"turn":1,"step":5}}
{"type":"assistant/message","data":{"turn":1,"step":5,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:12}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235068943,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235068943,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1790235068943,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235068943,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":5,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:12}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027860,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027860,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1790240027860,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027860,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":5,"callId":"cua-native-5","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}
{"type":"tool/result","data":{"turn":1,"step":5,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-5"},"toolCallId":"cua-native-5","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-2\",\"tree\":\"0 AXButton Increment\\n1 AXTextField value: \\n2 AXStaticText Count: 1\",\"elements\":[{\"index\":0,\"role\":\"AXButton\",\"title\":\"Increment\",\"bounds\":{\"x\":60,\"y\":80,\"width\":100,\"height\":20}}],\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:13}}"}},"sourceEventSeqs":[34],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":5}}
{"type":"step/start","data":{"turn":1,"step":6}}
{"type":"assistant/message","data":{"turn":1,"step":6,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:14}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235068959,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235068959,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1790235068959,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235068959,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":6,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:14}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027874,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027874,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1790240027874,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027874,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":6,"callId":"cua-native-6","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}
{"type":"tool/result","data":{"turn":1,"step":6,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-6"},"toolCallId":"cua-native-6","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"window-points\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:15}}"}},"sourceEventSeqs":[39],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":6}}
{"type":"step/start","data":{"turn":1,"step":7}}
{"type":"assistant/message","data":{"turn":1,"step":7,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:16}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235069016,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235069016,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1790235069016,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235069016,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":7,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:16}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027926,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027926,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1790240027926,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027926,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":7,"callId":"cua-native-7","name":"cua","arguments":"{\"method\":\"listWindows\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}
{"type":"tool/result","data":{"turn":1,"step":7,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-7"},"toolCallId":"cua-native-7","content":[{"type":"text","text":"[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true},{\"windowId\":8,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}]"}],"isError":false,"id":"{{message:17}}"}},"sourceEventSeqs":[44],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":7}}
{"type":"step/start","data":{"turn":1,"step":8}}
{"type":"assistant/message","data":{"turn":1,"step":8,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:18}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235069033,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235069033,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1790235069033,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235069033,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":8,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:18}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027941,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027941,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1790240027941,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027941,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":8,"callId":"cua-native-8","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}
{"type":"tool/result","data":{"turn":1,"step":8,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-8"},"toolCallId":"cua-native-8","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"tree\":\"\",\"elements\":[],\"windows\":[{\"windowId\":8,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:19}}"}},"sourceEventSeqs":[49],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":8}}
{"type":"step/start","data":{"turn":1,"step":9}}
{"type":"assistant/message","data":{"turn":1,"step":9,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:20}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235069048,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235069048,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1790235069048,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235069048,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":9,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:20}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027955,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027955,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1790240027955,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027955,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":9,"callId":"cua-native-9","name":"cua","arguments":"{\"method\":\"screenshot\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}
{"type":"tool/result","data":{"turn":1,"step":9,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-9"},"toolCallId":"cua-native-9","content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":8,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"window-points\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"isError":false,"id":"{{message:21}}"}},"sourceEventSeqs":[54],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":9}}
{"type":"step/start","data":{"turn":1,"step":10}}
{"type":"assistant/message","data":{"turn":1,"step":10,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":20,\"y\":20}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:22}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235069072,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235069072,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":20,\"y\":20}}"}}},{"type":"chunk","time":1790235069072,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235069072,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":10,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":20,\"y\":20}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:22}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027980,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027980,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":20,\"y\":20}}"}}},{"type":"chunk","time":1790240027980,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027980,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":10,"callId":"cua-native-10","name":"cua","arguments":"{\"method\":\"click\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8,\"snapshotId\":\"fixture-snapshot-3\",\"x\":20,\"y\":20}}"}}
{"type":"tool/result","data":{"turn":1,"step":10,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-10"},"toolCallId":"cua-native-10","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false}"}],"isError":false,"id":"{{message:23}}"}},"sourceEventSeqs":[59],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":10}}
{"type":"step/start","data":{"turn":1,"step":11}}
{"type":"assistant/message","data":{"turn":1,"step":11,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:24}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235069087,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790235069087,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1790235069087,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235069087,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":11,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:24}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240027996,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240027996,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}},{"type":"chunk","time":1790240027996,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240027996,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":11,"callId":"cua-native-11","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":8}}"}}
{"type":"tool/result","data":{"turn":1,"step":11,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-11"},"toolCallId":"cua-native-11","content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":8,\"snapshotId\":\"fixture-snapshot-4\",\"tree\":\"\",\"elements\":[],\"windows\":[{\"windowId\":8,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"AXless canvas\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}],"isError":false,"id":"{{message:25}}"}},"sourceEventSeqs":[64],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":11}}
{"type":"step/start","data":{"turn":1,"step":12}}
{"type":"assistant/message","data":{"turn":1,"step":12,"message":{"role":"assistant","content":[{"type":"text","text":"Observed Count: 1. Captured the AXless window and dispatched one coordinate click; its fresh AX state is still empty."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:26}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790235069102,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"chunk","time":1790235069102,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"Observed Count: 1. Captured the AXless window and dispatched one coordinate click; its fresh AX state is still empty."}}},{"type":"chunk","time":1790235069102,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790235069102,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"}
{"type":"assistant/message","data":{"turn":1,"step":12,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:26}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240028011,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240028011,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1790240028011,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240028011,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":12,"callId":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}
{"type":"tool/result","data":{"turn":1,"step":12,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-first"},"toolCallId":"cua-native-activate-first","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"foreground\":{\"requestId\":\"cua-native-activate-first\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:27}}"}},"sourceEventSeqs":[69],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":12}}
{"type":"step/start","data":{"turn":1,"step":13}}
{"type":"assistant/message","data":{"turn":1,"step":13,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:28}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240028025,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1790240028025,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1790240028025,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240028025,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
{"type":"tool/call","data":{"turn":1,"step":13,"callId":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}
{"type":"tool/result","data":{"turn":1,"step":13,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-repeat"},"toolCallId":"cua-native-activate-repeat","content":[{"type":"text","text":"{\"dispatched\":true,\"verified\":false,\"foreground\":{\"requestId\":\"cua-native-activate-repeat\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":false,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:29}}"}},"sourceEventSeqs":[74],"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":13}}
{"type":"step/start","data":{"turn":1,"step":14}}
{"type":"assistant/message","data":{"turn":1,"step":14,"message":{"role":"assistant","content":[{"type":"text","text":"Observed Count: 1. Captured the AXless window and dispatched one coordinate click; its fresh AX state is still empty."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:30}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1790240028039,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"chunk","time":1790240028039,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"Observed Count: 1. Captured the AXless window and dispatched one coordinate click; its fresh AX state is still empty."}}},{"type":"chunk","time":1790240028039,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1790240028039,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"}
{"type":"step/end","data":{"turn":1,"step":14}}
{"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}}