refactor(cua): centralize scoped surfaces and provider capabilities

This commit is contained in:
Huo Yaoyuan committed 2026-11-05 13:13:53 +08:00
1 parent 9aa3272fa5
commit ca0760bd1c
17 files changed
+317 -121

No files matched your search

@@ -191,12 +191,9 @@ actor AXWorker {
guard !closed else { throw NativeFailure(code: "NATIVE_CLOSED", message: "Helper is closing") }
let locked = observeSession()
if method == "getCapabilities" {
return .result(.object(["platform": .string("macos"), "permissions": .object([
return .result(.object(["permissions": .object([
"accessibility": .bool(AXIsProcessTrusted()), "screenRecording": .bool(CGPreflightScreenCaptureAccess())]),
"appLevelAccessibility": .bool(true),
"keyboard": .array([.string("app")]),
"sessionLocked": locked.map(JSONValue.bool) ?? .null,
"methods": .array(NativeProtocol.modelMethods.map(JSONValue.string)), "coordinateSpace": .string("window-points"), "browser": .bool(false)]))
"sessionLocked": locked.map(JSONValue.bool) ?? .null]))
}
if method == "listApps" { return .result(.array(catalog.applications.filter { $0.regular && $0.name != nil }.map(\.json))) }
try NativePlatform.requireUnlocked(locked)
@@ -1,6 +1,5 @@
// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.
enum NativeProtocol {
static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "invoke", "listActions", "listApps", "listWindows", "paste", "performAction", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"]
static let fields: [String: Set<String>] = [
"activate": ["appHandle"],
"applicationIcons": ["paths"],
@@ -11,23 +10,23 @@ enum NativeProtocol {
"getCapabilities": [],
"getState": ["appHandle", "details", "includeMenus", "maxDepth", "maxElements", "windowId"],
"hover": ["appHandle", "elementId", "snapshotId", "windowId", "x", "y"],
"invoke": ["appHandle", "elementId", "snapshotId"],
"invoke": ["appHandle", "elementId", "snapshotId", "windowId"],
"launchApp": ["appHandle"],
"listActions": ["appHandle", "elementId", "snapshotId"],
"listActions": ["appHandle", "elementId", "snapshotId", "windowId"],
"listApps": [],
"listWindows": ["appHandle"],
"openPermissionSettings": ["permission"],
"paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text"],
"performAction": ["action", "appHandle", "argument", "elementId", "snapshotId"],
"paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text", "windowId"],
"performAction": ["action", "appHandle", "argument", "elementId", "snapshotId", "windowId"],
"permissionStatus": [],
"pressKey": ["appHandle", "key", "modifiers"],
"pressKey": ["appHandle", "key", "modifiers", "windowId"],
"requestPermission": ["permission"],
"screenshot": ["appHandle", "windowId"],
"scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"],
"selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text"],
"setValue": ["appHandle", "elementId", "snapshotId", "value"],
"selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text", "windowId"],
"setValue": ["appHandle", "elementId", "snapshotId", "value", "windowId"],
"show": ["appHandle", "windowId"],
"typeText": ["appHandle", "text"],
"typeText": ["appHandle", "text", "windowId"],
]
static let stringEnums: [String: [String: Set<String>]] = [
"click": ["modifiers": ["command", "control", "fn", "option", "shift"], "mouseButton": ["left", "middle", "right"]],
@@ -7,12 +7,11 @@ import { nativeProtocol } from '@deepseek-ai/dsh-cua/native-schema'
* @returns Native JSON Schema and Swift field/enum tables, including private continuations.
*/
export function renderNativeProtocol(): { json: string; swift: string } {
const { fields, enums, modelMethods, json } = nativeProtocol()
const { fields, enums, json } = nativeProtocol()
const quoted = (values: Iterable<string>) => [...values].sort().map(value => JSON.stringify(value)).join(', ')
const swift = [
'// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.',
'enum NativeProtocol {',
` static let modelMethods: [String] = [${quoted(modelMethods)}]`,
' static let fields: [String: Set<String>] = [',
...[...fields].sort(([a], [b]) => a.localeCompare(b)).map(([method, names]) => ` ${JSON.stringify(method)}: [${quoted(names)}],`),
' ]',
@@ -290,6 +290,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"snapshotId": {
"type": "string",
"minLength": 1
@@ -332,6 +337,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"snapshotId": {
"type": "string",
"minLength": 1
@@ -415,6 +425,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"snapshotId": {
"type": "string",
"minLength": 1
@@ -459,6 +474,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"snapshotId": {
"type": "string",
"minLength": 1
@@ -538,6 +558,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"snapshotId": {
"type": "string",
"minLength": 1
@@ -693,6 +718,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"snapshotId": {
"type": "string",
"minLength": 1
@@ -772,6 +802,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"snapshotId": {
"type": "string",
"minLength": 1
@@ -833,6 +868,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"text": {
"type": "string"
}
@@ -868,6 +908,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"text": {
"type": "string"
},
@@ -934,6 +979,11 @@
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"key": {
"type": "string",
"minLength": 1
@@ -144,7 +144,6 @@ def main():
call = helper.call
capabilities = call("getCapabilities", {})
assert capabilities["permissions"] == {"accessibility": True, "screenRecording": True}, capabilities
assert "launchApp" not in capabilities["methods"], capabilities
matches = call("findApps", {"query": str(application), "maxAppMatches": 20})
assert len(matches) == 1 and matches[0]["isRunning"] is False, matches
pending = call("getApp", {"selector": str(application)})
@@ -126,7 +126,6 @@ def main():
time.sleep(0.025)
capabilities = call("getCapabilities", {})
assert capabilities["platform"] == "macos", capabilities
assert capabilities["permissions"] == {"accessibility": True, "screenRecording": True}, capabilities
apps = call("listApps", {})
assert any(app["pid"] == fixture.pid for app in apps), apps
+18 -2
View File
@@ -3,11 +3,25 @@ import { isAbsolute, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import type { Context } from '@deepseek-ai/cordis'
import { CuaConfig, installCua } from '@deepseek-ai/dsh-cua'
import type { CuaStartupCapabilities } from '@deepseek-ai/dsh-cua'
import { nativeHostArchitecture, prepareNativeExecutable } from './native-build.ts'
import { createSharedNativeChannel } from './shared-native.ts'
import { GUIDANCE } from './guidance.ts'
export { GUIDANCE } from './guidance.ts'
const CAPABILITIES: CuaStartupCapabilities = {
platform: 'macos',
methods: [
'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'hover', 'invoke',
'listActions', 'listApps', 'listWindows', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll',
'selectText', 'setValue', 'show', 'typeText',
],
appLevelAccessibility: true,
keyboard: ['app'],
browser: false,
coordinateSpace: 'screenshot-pixels',
}
/** Loader plugin identity. */
export const name = 'experimental-cua-macos'
/** Services required by the shared CUA executor and the native launcher. */
@@ -27,7 +41,9 @@ export const Config: typeof CuaConfig = CuaConfig
export async function apply(ctx: Context, config: Config): Promise<void> {
if (config.helperPath !== undefined) {
if (!isAbsolute(config.helperPath)) throw new Error('dsh-cua helperPath must be absolute')
installCua(ctx, config, { helperPath: config.helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE })
installCua(ctx, config, {
helperPath: config.helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE, capabilities: CAPABILITIES,
})
return
}
const architecture = nativeHostArchitecture(process.platform, process.arch)
@@ -44,5 +60,5 @@ export async function apply(ctx: Context, config: Config): Promise<void> {
}, 'dsh-cua.native-preparation')
const helperPath = await prepared
controller.signal.throwIfAborted()
installCua(ctx, config, { helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE })
installCua(ctx, config, { helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE, capabilities: CAPABILITIES })
}
@@ -162,13 +162,18 @@ it('correlates a native request with its logged tool invocation', async () => {
expect(native.request).toHaveBeenCalledExactlyOnceWith('listApps', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1'))
})
it('reports screenshot pixel coordinates while preserving native capabilities', async () => {
it('merges startup capabilities into runtime native fields', async () => {
const f = await setup()
const capabilities = { platform: 'macos', coordinateSpace: 'window-points', permissions: { accessibility: true, screenCapture: false }, appLevelAccessibility: true, keyboard: ['app'] }
native.request.mockResolvedValueOnce(capabilities)
const runtime = { permissions: { accessibility: true, screenRecording: false }, sessionLocked: false }
native.request.mockResolvedValueOnce(runtime)
const result = await f.execute('getCapabilities')
expect(result.isError).toBe(false)
expect(result.value).toEqual({ ...capabilities, coordinateSpace: 'screenshot-pixels' })
expect(result.value).toEqual({ ...runtime, platform: 'macos', appLevelAccessibility: true, keyboard: ['app'], browser: false,
coordinateSpace: 'screenshot-pixels', methods: [
'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'hover', 'invoke',
'listActions', 'listApps', 'listWindows', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll',
'selectText', 'setValue', 'show', 'typeText',
] })
expect(native.request).toHaveBeenCalledExactlyOnceWith('getCapabilities', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1'))
})
+102 -64
View File
@@ -3,7 +3,7 @@ import { Buffer } from 'node:buffer'
import type { z } from 'zod'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts'
import type { CuaApi, CuaApp, CuaAppHandle, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts'
import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts'
export type * from './types.ts'
function requestIssueMessage(issues: readonly z.core.$ZodIssue[], parent: readonly PropertyKey[] = []): string {
@@ -56,10 +56,6 @@ function validatedCall(dispatch: CuaDispatch) {
}
}
async function unsupportedKeyboardInput(): Promise<never> {
throw new Error('Keyboard input is not supported at this scope.')
}
async function emit(sink: CuaObservationSink | undefined, value: object, options?: CuaObservationOptions): Promise<void> {
if (options?.emit !== false) await sink?.text(JSON.stringify(value))
}
@@ -72,11 +68,17 @@ async function emit(sink: CuaObservationSink | undefined, value: object, options
*/
export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): CuaApi {
const call = validatedCall(dispatch)
const objects = createCuaObjects(dispatch, sink)
let capabilities: CuaCapabilities | undefined
async function currentCapabilities(): Promise<CuaCapabilities> {
if (capabilities === undefined) capabilities = CapabilitiesResult.parse(await call('getCapabilities', {}))
return capabilities
}
const objects = createCuaObjects(call, sink)
const api: CuaApi = {
async getCapabilities(options) {
const result = CapabilitiesResult.parse(await call('getCapabilities', {}))
capabilities = result
await emit(sink, result, options)
return result
},
@@ -98,82 +100,112 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
return result
},
async getApp(selector) {
const request = Request.safeParse({ method: 'getApp', params: { selector } })
if (!request.success) throw new Error(`getApp ${requestIssueMessage(request.error.issues)}`)
const capabilities = await currentCapabilities()
const { appHandle } = AppResult.parse(await call('getApp', { selector }))
const app = objects.createApp(appHandle)
await app.getAccessibilityTree({ emit: false })
const app = objects.createCuaApp(appHandle, capabilities)
if (capabilities.appLevelAccessibility) await app.getAccessibilityTree({ emit: false })
return app
},
}
return api
}
function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | undefined): {
createApp(appHandle: CuaAppHandle): CuaApp
createWindow(appHandle: CuaAppHandle, windowId: number): CuaWindow
type CuaScopeParams = (params: Record<string, JsonValue>) => Record<string, JsonValue>
function createCuaObjects(call: ReturnType<typeof validatedCall>, sink: CuaObservationSink | undefined): {
createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp
} {
const call = validatedCall(dispatch)
const action = async (method: CuaOperation['method'], params: Record<string, JsonValue>) => ActionResult.parse(await call(method, params))
const observed = new Map<CuaAppHandle, { snapshotId: string; windowId: number | undefined }>()
function elementParams(appHandle: CuaAppHandle): Record<string, JsonValue> {
function currentSnapshot(appHandle: CuaAppHandle): string {
const current = observed.get(appHandle)
if (current === undefined) throw new Error('Read getAccessibilityTree before using this surface')
return { appHandle, snapshotId: current.snapshotId }
return current.snapshotId
}
function coordinateParams(appHandle: CuaAppHandle, windowId: number): Record<string, JsonValue> {
const current = observed.get(appHandle)
return {
appHandle, windowId,
...current !== undefined && (current.windowId === undefined || current.windowId === windowId)
? { snapshotId: current.snapshotId } : {},
}
return current !== undefined && (current.windowId === undefined || current.windowId === windowId)
? { snapshotId: current.snapshotId } : {}
}
function createSurface(appHandle: CuaAppHandle, selectedWindow?: number): CuaSurface {
return {
async show() {
return ShowResult.parse(await call('show', { appHandle,
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
}))
},
function createCuaSurface(
target: { appHandle: CuaAppHandle; windowId?: number },
paramsFor: CuaScopeParams,
requireAccessibility: () => void,
requireKeyboard: () => void,
) {
const scopedCall = (method: CuaOperation['method'], params: Record<string, JsonValue>) =>
call(method, paramsFor(params))
const action = async (method: CuaOperation['method'], params: Record<string, JsonValue>) =>
ActionResult.parse(await scopedCall(method, params))
const elementParams = (): Record<string, JsonValue> => ({ snapshotId: currentSnapshot(target.appHandle) })
const surface: CuaSurface = {
async show() { return ShowResult.parse(await scopedCall('show', {})) },
async getAccessibilityTree(options) {
observed.delete(appHandle)
const state = StateResult.parse(await call('getState', { appHandle,
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
requireAccessibility()
observed.delete(target.appHandle)
const state = StateResult.parse(await scopedCall('getState', {
...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus },
...options?.details === undefined ? {} : { details: options.details },
}))
observed.set(appHandle, { snapshotId: state.snapshotId, windowId: selectedWindow })
observed.set(target.appHandle, { snapshotId: state.snapshotId, windowId: target.windowId })
if (options?.emit !== false) await sink?.text(state.tree)
return state.tree
},
async listActions(elementId) {
return ActionNamesResult.parse(await call('listActions', { ...elementParams(appHandle), elementId }))
requireAccessibility()
return ActionNamesResult.parse(await scopedCall('listActions', { ...elementParams(), elementId }))
},
async performAction(elementId, name, argument) {
return action('performAction', { ...elementParams(appHandle), elementId, action: name,
requireAccessibility()
return action('performAction', { ...elementParams(), elementId, action: name,
...argument === undefined ? {} : { argument },
})
},
async invoke(elementId) {
return ActionResult.parse(await call('invoke', { ...elementParams(appHandle), elementId }))
requireAccessibility()
return action('invoke', { ...elementParams(), elementId })
},
async setValue(elementId, value) {
requireAccessibility()
return action('setValue', { ...elementParams(), elementId, value })
},
async setValue(elementId, value) { return action('setValue', { ...elementParams(appHandle), elementId, value }) },
async selectText(elementId, text, options) {
return action('selectText', { ...elementParams(appHandle), elementId, text,
requireAccessibility()
return action('selectText', { ...elementParams(), elementId, text,
...options?.prefix === undefined ? {} : { prefix: options.prefix },
...options?.suffix === undefined ? {} : { suffix: options.suffix },
...options?.selectionType === undefined ? {} : { selectionType: options.selectionType },
})
},
typeText: unsupportedKeyboardInput,
paste: unsupportedKeyboardInput,
pressKey: unsupportedKeyboardInput,
async typeText(text) {
requireKeyboard()
return action('typeText', { text })
},
async paste(text, options) {
requireKeyboard()
return action('paste', { text,
...options?.format === undefined ? {} : { format: options.format },
})
},
async pressKey(key) {
requireKeyboard()
return action('pressKey', keyChord(key))
},
}
return { surface, call: scopedCall, action, elementParams }
}
function createWindow(appHandle: CuaAppHandle, windowId: number): CuaWindow {
function createCuaWindow(appHandle: CuaAppHandle, windowId: number, capabilities: CuaCapabilities): CuaWindow {
const base = createCuaSurface({ appHandle, windowId },
params => ({ ...params, appHandle, windowId }), () => undefined, () => {
if (!capabilities.keyboard.includes('window')) {
throw new Error('Window-scoped keyboard input is not supported on current platform.')
}
})
return {
...createSurface(appHandle, windowId), windowId,
...base.surface, windowId,
async getScreenshot(options) {
const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId }))
const image = ScreenshotResult.parse(await base.call('screenshot', {}))
const screenshot: CuaScreenshot = {
bytes: Uint8Array.from(Buffer.from(image.data, 'base64')),
mimeType: image.mimeType,
@@ -184,37 +216,39 @@ function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | unde
if (options?.emit !== false) await sink?.image(screenshot)
return screenshot
},
async hover(target, options) {
async hover(targetPoint, options) {
const relative = options?.relative === undefined ? {} : { relative: options.relative }
if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...elementParams(appHandle), elementId: target, ...relative }))
const [x, y] = target
return HoverResult.parse(await call('hover', { ...coordinateParams(appHandle, windowId), x, y, ...relative }))
if (typeof targetPoint === 'string') {
return HoverResult.parse(await base.call('hover', { ...base.elementParams(), elementId: targetPoint, ...relative }))
}
const [x, y] = targetPoint
return HoverResult.parse(await base.call('hover', { ...coordinateParams(appHandle, windowId), x, y, ...relative }))
},
async click(target, options) {
async click(targetPoint, options) {
const mouseButton = options?.mouseButton ?? 'left'
const clickCount = options?.clickCount ?? 1
if (mouseButton !== 'left' && clickCount !== 1) throw new Error('Only left double-click is supported')
const modifiers = options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] }
const hold = options?.holdMs === undefined ? {} : { holdMs: options.holdMs }
const relative = options?.relative === undefined ? {} : { relative: options.relative }
if (typeof target === 'string') {
return action('click', { ...elementParams(appHandle), ...modifiers, ...hold, ...relative, elementId: target,
if (typeof targetPoint === 'string') {
return base.action('click', { ...base.elementParams(), ...modifiers, ...hold, ...relative, elementId: targetPoint,
...mouseButton === 'left' ? {} : { mouseButton }, ...clickCount === 1 ? {} : { clickCount },
})
}
const [x, y] = target
return action('click', { ...coordinateParams(appHandle, windowId), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative })
const [x, y] = targetPoint
return base.action('click', { ...coordinateParams(appHandle, windowId), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative })
},
async scroll(options) {
if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero')
return action('scroll', {
return base.action('scroll', {
...coordinateParams(appHandle, windowId),
x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY,
...options.relative === undefined ? {} : { relative: options.relative },
})
},
async drag(from, to, options) {
return action('drag', { ...coordinateParams(appHandle, windowId),
return base.action('drag', { ...coordinateParams(appHandle, windowId),
fromX: from[0], fromY: from[1], toX: to[0], toY: to[1],
...options?.relative === undefined ? {} : { relative: options.relative },
...options?.mouseButton === undefined ? {} : { mouseButton: options.mouseButton },
@@ -223,26 +257,30 @@ function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | unde
},
}
}
function createApp(appHandle: CuaAppHandle): CuaApp {
function createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp {
const base = createCuaSurface({ appHandle }, params => ({ ...params, appHandle }), () => {
if (!capabilities.appLevelAccessibility) {
throw new Error('Application-level accessibility is not supported on current platform; observe a window instead.')
}
}, () => {
if (!capabilities.keyboard.includes('app')) {
throw new Error('App-scoped keyboard input is not supported on current platform.')
}
})
return {
...createSurface(appHandle), appHandle,
async activate() { return action('activate', { appHandle }) },
...base.surface, appHandle,
async activate() { return base.action('activate', {}) },
async getWindows(options) {
const windows = WindowsResult.parse(await call('listWindows', { appHandle }))
const windows = WindowsResult.parse(await base.call('listWindows', {}))
await emit(sink, windows, options)
return windows
},
async getWindow(windowId): Promise<CuaWindow> {
const window = createWindow(appHandle, windowId)
const window = createCuaWindow(appHandle, windowId, capabilities)
await window.getAccessibilityTree({ emit: false })
return window
},
async typeText(text) { return action('typeText', { appHandle, text }) },
async paste(text, options) { return action('paste', { appHandle, text,
...options?.format === undefined ? {} : { format: options.format },
}) },
async pressKey(key) { return action('pressKey', { appHandle, ...keyChord(key) }) },
}
}
return { createApp, createWindow }
return { createCuaApp }
}
+17 -3
View File
@@ -16,7 +16,7 @@ import { NativeClient, NativeError, type NativeClientConfig, type NativeChannelF
import { CuaPermissionStore } from './permission-store.ts'
import { ApplicationAuthorizationResult, installCuaAccess } from './access.ts'
import type { CuaApplicationGrant, CuaApplicationId } from './access-types.ts'
import type { CuaAppHandle, CuaOperation } from './types.ts'
import type { CuaAppHandle, CuaOperation, CuaStartupCapabilities } from './types.ts'
import { ApplicationIcon, AppResult, NativeScreenshotResult, Request, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts'
import { CuaCoordinates } from './coordinates.ts'
import { createScope, type Scope } from '@deepseek-ai/dsh-scope'
@@ -29,6 +29,7 @@ import type {} from '@deepseek-ai/dsh-ptc-runtime'
import type {} from '@deepseek-ai/cordis-plugin-loader'
export { CuaAccess } from './access.ts'
export type { CuaStartupCapabilities } from './types.ts'
/** Native executable and per-operation resource bounds. */
export interface CuaConfig {
@@ -145,11 +146,12 @@ async function screenshotStore(ctx: Context, execution: Readonly<CuaExecution>)
return { attachments, target: (image: ImageAttachmentRef) => llm.imageRequestTarget(provider, model, image) }
}
/** Prepared platform executable, transport, and platform-specific operating instructions. */
/** Prepared platform executable, transport, platform-specific instructions and startup capabilities. */
export interface CuaBackend {
helperPath: string
createChannel: NativeChannelFactory
guidance: string
capabilities: CuaStartupCapabilities
}
/**
@@ -282,7 +284,19 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend)
return selected
}
signal.throwIfAborted()
if (method === 'getCapabilities') return { ...z.record(z.string(), z.json()).parse(result), coordinateSpace: 'screenshot-pixels' }
if (method === 'getCapabilities') {
const runtime = z.record(z.string(), z.json()).parse(result)
const capabilities = backend.capabilities
return {
...runtime,
platform: capabilities.platform,
methods: [...capabilities.methods],
appLevelAccessibility: capabilities.appLevelAccessibility,
keyboard: [...capabilities.keyboard],
browser: capabilities.browser,
coordinateSpace: capabilities.coordinateSpace,
}
}
if (method === 'getState') {
const state = StateResult.parse(result)
execution.coordinates.observe(state.appHandle, state.windows, state.windowId ?? undefined)
@@ -13,16 +13,14 @@ function variants(schema: Schema): Schema[] { return schema.anyOf ?? schema.oneO
/**
* Derive native request schemas and admission fields from the shared operation definitions.
* @returns Model operation names, native field and enum tables, and the complete wire schema.
* @returns Native field and enum tables and the complete wire schema.
*/
export function nativeProtocol(): {
fields: Map<string, Set<string>>
enums: Map<string, Map<string, Set<string>>>
modelMethods: string[]
json: string
} {
const resources = new Map(Object.entries(NativeResourceParameters))
const modelMethods = Request.options.map(schema => schema.shape.method.value).sort()
const schemas = [...Request.options, ...NativeOnlyOperations]
.map(schema => z.toJSONSchema(schema))
const fields = new Map<string, Set<string>>()
@@ -69,7 +67,7 @@ export function nativeProtocol(): {
fields.set(method, names)
if (values.size) enums.set(method, values)
}
return { fields, enums, modelMethods, json: JSON.stringify({
return { fields, enums, json: JSON.stringify({
$schema: 'https://json-schema.org/draft/2020-12/schema',
$comment: 'Generated from the shared CUA operation schema. Do not edit.',
title: 'dsh-cua native request',
+4 -4
View File
@@ -17,7 +17,7 @@ const window = { ...app, windowId: id }
const jsonValue = z.json()
export const elementId = z.string().min(1)
.describe('Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot.')
const snapshot = { ...app, snapshotId: z.string().min(1), elementId }
const snapshot = { ...app, windowId: id.optional(), snapshotId: z.string().min(1), elementId }
const coordinate = { ...window, snapshotId: z.string().min(1).optional(), x: point, y: point }
const absoluteCoordinate = { ...coordinate, relative: z.literal(false).optional().describe('Omitted or false uses pixels in the latest screenshot returned by this tool.') }
const relativeCoordinate = { ...coordinate, relative: z.literal(true).describe('Use integer thousandths of the window width and height, from 0 to 999.'), x: relativePoint, y: relativePoint }
@@ -59,9 +59,9 @@ export const Request = z.discriminatedUnion('method', [
operation('setValue', { ...snapshot, value: jsonValue }),
operation('selectText', { ...snapshot, text: z.string().min(1), prefix: z.string().optional(), suffix: z.string().optional(),
selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }),
operation('typeText', { ...app, text: z.string() }),
operation('paste', { ...app, text: z.string(), format: z.enum(['text', 'html']).optional() }),
operation('pressKey', { ...app, key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }),
operation('typeText', { ...app, windowId: id.optional(), text: z.string() }),
operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }),
operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }),
z.object({ method: z.literal('scroll'), params: z.union([
z.object({ ...absoluteCoordinate,
deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(),
+5
View File
@@ -45,6 +45,11 @@ export interface CuaAxOptions extends CuaObservationOptions {
}
/** Reported permissions and operations; null sessionLocked means no usable console session was verified. */
export type CuaCapabilities = z.output<typeof CapabilitiesResult>
/** Provider capabilities fixed when the provider is installed; runtime fields come from getCapabilities. */
export type CuaStartupCapabilities = Pick<
CuaCapabilities,
'platform' | 'methods' | 'appLevelAccessibility' | 'keyboard' | 'browser' | 'coordinateSpace'
>
/** App discovery result accepted by getApp. */
export type CuaAppInfo = Pick<Extract<CuaFoundAppInfo, { isRunning: true }>, 'id' | 'displayName' | 'isRunning' | 'pid'>
/** An installed application match; running matches include a process identifier. */
@@ -19,6 +19,10 @@ const provider = {
helperPath: resolve('unused-cua-helper'),
createChannel() { throw new Error('Scope tests must not start a native helper') },
guidance: 'Fixture CUA guidance',
capabilities: {
platform: 'macos', methods: ['getCapabilities', 'getApp', 'getState'], appLevelAccessibility: true,
keyboard: ['app'], browser: false, coordinateSpace: 'screenshot-pixels',
},
})
},
}
+83 -17
View File
@@ -4,7 +4,7 @@ import { describe, expect, expectTypeOf, it, vi } from 'vitest'
import { brandString } from '@deepseek-ai/dsh-brand'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import { createCuaApi } from '../src/api.ts'
import type { CuaApp, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts'
import type { CuaApp, CuaCapabilities, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts'
/** Element ids are opaque strings; these fixtures use the macOS decimal form. */
const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
@@ -13,7 +13,7 @@ const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG
const bounds = { x: -640, y: 100, width: 640, height: 480 }
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', running: true }
const delivered = { dispatched: true }
const capabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
const capabilities: CuaCapabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
function windowInfo(windowId: number) {
return { windowId, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null }
@@ -21,11 +21,12 @@ function windowInfo(windowId: number) {
function fixture(withSink = true) {
let windows = [windowInfo(23)]
let currentCapabilities: CuaCapabilities = capabilities
let generation = 0
const sink = { text: vi.fn<(value: string) => void>(), image: vi.fn<(screenshot: CuaScreenshot) => void>() }
const dispatch = vi.fn(async ({ method, params }: CuaOperation): Promise<JsonValue> => {
switch (method) {
case 'getCapabilities': return capabilities
case 'getCapabilities': return currentCapabilities
case 'listApps': return [selected, { pid: 18, name: 'Utility' }]
case 'getApp': return selected
case 'listWindows': return windows
@@ -45,6 +46,7 @@ function fixture(withSink = true) {
dispatch, sink, api: createCuaApi(dispatch, withSink ? sink : undefined),
setWindows: (ids: number[]) => { windows = ids.map(windowInfo) },
setWindowRecords: (records: ReturnType<typeof windowInfo>[]) => { windows = records },
setCapabilities: (value: CuaCapabilities) => { currentCapabilities = value },
}
}
@@ -163,7 +165,7 @@ describe('hover across observations', () => {
await window.getScreenshot({ emit: false })
expect(await window.hover([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0' } },
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: 23 } },
{ method: 'getState', params: { appHandle: 'app-17', windowId: 23 } },
{ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } },
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 23, x: 30, y: 40 } },
@@ -199,7 +201,7 @@ describe('pointer buttons and modifiers', () => {
const window = await app.getWindow(23)
await window.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', mouseButton: 'middle', modifiers: ['shift'],
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: 23, mouseButton: 'middle', modifiers: ['shift'],
} })
await window.click([10, 20], { modifiers: ['command'] })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
@@ -322,18 +324,18 @@ describe('application and window observations', () => {
await second.click(element('3'))
expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '1', value: 'First document' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2', windowId: 23 } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3', windowId: 24 } },
])
expect(first.windowId).toBe(23)
expect(second.windowId).toBe(24)
expect(f.sink.text.mock.calls.map(([text]) => text)).toEqual([])
await expect(second.getAccessibilityTree({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4')
await second.click(element('3'))
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3' } })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3', windowId: 24 } })
await app.getAccessibilityTree({ emit: false })
await first.invoke(element('4'))
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4' } })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4', windowId: 23 } })
expect(f.sink.text).not.toHaveBeenCalled()
})
@@ -476,6 +478,69 @@ describe('application and window observations', () => {
await expect(app.getWindows()).resolves.toEqual([observed])
})
it('skips app-level accessibility while keeping window observation available', async () => {
const f = fixture()
f.setCapabilities({ ...capabilities, appLevelAccessibility: false })
const app = await f.api.getApp('Editor')
expect(f.dispatch.mock.calls.map(([call]) => call.method)).toEqual(['getCapabilities', 'getApp'])
await expect(app.getAccessibilityTree()).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
await expect(app.listActions(element('0'))).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
await expect(app.performAction(element('0'), 'AXPress')).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
await expect(app.invoke(element('0'))).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
await expect(app.setValue(element('0'), 'value')).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
await expect(app.selectText(element('0'), 'text')).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
expect(f.dispatch).toHaveBeenCalledTimes(2)
f.dispatch.mockClear()
await expect(app.typeText('keyboard without accessibility')).resolves.toEqual(delivered)
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'typeText', params: { appHandle: 'app-17', text: 'keyboard without accessibility' } },
])
const window = await app.getWindow(23)
await expect(window.getAccessibilityTree({ emit: false })).resolves.toBe('Tree 2')
await expect(app.getAccessibilityTree()).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
await expect(window.invoke(element('0'))).resolves.toEqual(delivered)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: {
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', elementId: '0',
} })
})
it('honors window keyboard support and keeps app keyboard available', async () => {
const f = fixture()
f.setCapabilities({ ...capabilities, keyboard: ['app', 'window'] })
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
f.dispatch.mockClear()
await window.typeText('hello')
await window.paste('<b>bold</b>', { format: 'html' })
await window.pressKey('cmd+a')
await app.typeText('app text')
await app.paste('app paste')
await app.pressKey('shift+Tab')
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'hello' } },
{ method: 'paste', params: { appHandle: 'app-17', windowId: 23, text: '<b>bold</b>', format: 'html' } },
{ method: 'pressKey', params: { appHandle: 'app-17', windowId: 23, key: 'a', modifiers: ['command'] } },
{ method: 'typeText', params: { appHandle: 'app-17', text: 'app text' } },
{ method: 'paste', params: { appHandle: 'app-17', text: 'app paste' } },
{ method: 'pressKey', params: { appHandle: 'app-17', key: 'Tab', modifiers: ['shift'] } },
])
})
it('rejects app keyboard when only window scope is supported', async () => {
const f = fixture()
f.setCapabilities({ ...capabilities, keyboard: ['window'] })
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
f.dispatch.mockClear()
await expect(app.typeText('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
await expect(app.paste('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
await expect(app.pressKey('Return')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
await expect(app.pressKey('unknown+a')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
expect(f.dispatch).not.toHaveBeenCalled()
await expect(window.typeText('window text')).resolves.toEqual(delivered)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'window text' } })
})
it('accepts coordinate input without a current snapshot', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
@@ -686,8 +751,9 @@ describe('input actions', () => {
() => window.typeText('hello'),
() => window.paste('hello'),
() => window.pressKey('Return'),
() => window.pressKey('unknown+a'),
]) {
await expect(run()).rejects.toThrow('Keyboard input is not supported at this scope.')
await expect(run()).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.')
}
expect(f.dispatch).not.toHaveBeenCalled()
})
@@ -699,8 +765,8 @@ describe('input actions', () => {
await window.click(element('12'), { mouseButton: 'right' })
await window.click(element('13'), { clickCount: 2 })
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', mouseButton: 'right' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', clickCount: 2 } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', windowId: 23, mouseButton: 'right' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', windowId: 23, clickCount: 2 } },
])
})
@@ -711,15 +777,15 @@ describe('input actions', () => {
await window.click(element('42.12345'))
await window.hover(element('42.12345'))
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } },
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: 23 } },
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: 23 } },
])
})
it('does not activate on selection and dispatches only an explicit activation request', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
expect(f.dispatch.mock.calls.map(([call]) => call.method)).toEqual(['getApp', 'getState'])
expect(f.dispatch.mock.calls.map(([call]) => call.method)).toEqual(['getCapabilities', 'getApp', 'getState'])
await app.activate()
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'activate', params: { appHandle: 'app-17' } })
})
@@ -731,10 +797,10 @@ describe('input actions', () => {
const failure = new Error('Application was restarted; select it again')
f.dispatch.mockRejectedValueOnce(failure)
await expect(app.typeText('hello')).rejects.toBe(failure)
expect(f.dispatch).toHaveBeenCalledTimes(4)
expect(f.dispatch).toHaveBeenCalledTimes(5)
f.dispatch.mockResolvedValueOnce({ dispatched: 'yes' })
await expect(window.click(element('0'))).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(5)
expect(f.dispatch).toHaveBeenCalledTimes(6)
})
it('rejects invalid selection inputs before dispatch', async () => {
+11 -4
View File
@@ -5,12 +5,17 @@ import { brandString } from '@deepseek-ai/dsh-brand'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import { connectCua } from '../src/bridge.ts'
import type { CuaToolBinding } from '../src/bridge.ts'
import type { CuaElementId, CuaOperation } from '../src/types.ts'
import type { CuaCapabilities, CuaElementId, CuaOperation } from '../src/types.ts'
/** Element ids are opaque strings; these fixtures use the macOS decimal form. */
const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC'
const capabilities: CuaCapabilities = {
platform: 'macos', permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: true,
keyboard: ['app'], sessionLocked: false, methods: ['getCapabilities', 'getApp', 'getState', 'screenshot'],
coordinateSpace: 'screenshot-pixels', browser: false,
}
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', running: true }
const bounds = { x: 0, y: 0, width: 640, height: 480 }
const snapshot = {
@@ -20,6 +25,7 @@ const snapshot = {
it('uses the current tool binding when a retained app is called by another execution', async () => {
const first = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => {
if (operation.method === 'getCapabilities') return capabilities
if (operation.method === 'getApp') return selected
if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null }
return snapshot
@@ -33,8 +39,8 @@ it('uses the current tool binding when a retained app is called by another execu
current = second
await expect(window.click(element('0'))).resolves.toEqual({ dispatched: true })
expect(first).toHaveBeenCalledTimes(3)
expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } })
expect(first).toHaveBeenCalledTimes(4)
expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', windowId: 23 } })
expect(text).not.toHaveBeenCalled()
current = first
await app.getAccessibilityTree()
@@ -61,6 +67,7 @@ it.each([null, 'text only', { content: [{ type: 'text', text: '{}' }] }])('rejec
it('returns screenshot bytes and metadata with one image-bearing tool call and no image text output', async () => {
const tool = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => {
if (operation.method === 'getCapabilities') return capabilities
if (operation.method === 'getApp') return selected
if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null }
return {
@@ -73,6 +80,6 @@ it('returns screenshot bytes and metadata with one image-bearing tool call and n
text.mockClear()
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getApp', 'getState', 'getState', 'screenshot'])
expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getCapabilities', 'getApp', 'getState', 'getState', 'screenshot'])
expect(text).not.toHaveBeenCalled()
})
@@ -33,7 +33,7 @@ it('retains optional resource fields and tuple-valued parameters without inventi
})
it('admits only locator fields for default invocation', () => {
expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId']))
expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId', 'windowId']))
})
it('accepts a parameter-free object schema without a properties dictionary', () => {