diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index 3a1807127d..4773011b4e 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -191,12 +191,9 @@ actor AXWorker { guard !closed else { throw NativeFailure(code: "NATIVE_CLOSED", message: "Helper is closing") } let locked = observeSession() if method == "getCapabilities" { - return .result(.object(["platform": .string("macos"), "permissions": .object([ + return .result(.object(["permissions": .object([ "accessibility": .bool(AXIsProcessTrusted()), "screenRecording": .bool(CGPreflightScreenCaptureAccess())]), - "appLevelAccessibility": .bool(true), - "keyboard": .array([.string("app")]), - "sessionLocked": locked.map(JSONValue.bool) ?? .null, - "methods": .array(NativeProtocol.modelMethods.map(JSONValue.string)), "coordinateSpace": .string("window-points"), "browser": .bool(false)])) + "sessionLocked": locked.map(JSONValue.bool) ?? .null])) } if method == "listApps" { return .result(.array(catalog.applications.filter { $0.regular && $0.name != nil }.map(\.json))) } try NativePlatform.requireUnlocked(locked) diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift index 906dab51a8..543dca7189 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift @@ -1,6 +1,5 @@ // Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit. enum NativeProtocol { - static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "invoke", "listActions", "listApps", "listWindows", "paste", "performAction", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"] static let fields: [String: Set] = [ "activate": ["appHandle"], "applicationIcons": ["paths"], @@ -11,23 +10,23 @@ enum NativeProtocol { "getCapabilities": [], "getState": ["appHandle", "details", "includeMenus", "maxDepth", "maxElements", "windowId"], "hover": ["appHandle", "elementId", "snapshotId", "windowId", "x", "y"], - "invoke": ["appHandle", "elementId", "snapshotId"], + "invoke": ["appHandle", "elementId", "snapshotId", "windowId"], "launchApp": ["appHandle"], - "listActions": ["appHandle", "elementId", "snapshotId"], + "listActions": ["appHandle", "elementId", "snapshotId", "windowId"], "listApps": [], "listWindows": ["appHandle"], "openPermissionSettings": ["permission"], - "paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text"], - "performAction": ["action", "appHandle", "argument", "elementId", "snapshotId"], + "paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text", "windowId"], + "performAction": ["action", "appHandle", "argument", "elementId", "snapshotId", "windowId"], "permissionStatus": [], - "pressKey": ["appHandle", "key", "modifiers"], + "pressKey": ["appHandle", "key", "modifiers", "windowId"], "requestPermission": ["permission"], "screenshot": ["appHandle", "windowId"], "scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"], - "selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text"], - "setValue": ["appHandle", "elementId", "snapshotId", "value"], + "selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text", "windowId"], + "setValue": ["appHandle", "elementId", "snapshotId", "value", "windowId"], "show": ["appHandle", "windowId"], - "typeText": ["appHandle", "text"], + "typeText": ["appHandle", "text", "windowId"], ] static let stringEnums: [String: [String: Set]] = [ "click": ["modifiers": ["command", "control", "fn", "option", "shift"], "mouseButton": ["left", "middle", "right"]], diff --git a/packages/experimental/cua-macos/native/generate-protocol.ts b/packages/experimental/cua-macos/native/generate-protocol.ts index 01746dea68..1b0de044c0 100644 --- a/packages/experimental/cua-macos/native/generate-protocol.ts +++ b/packages/experimental/cua-macos/native/generate-protocol.ts @@ -7,12 +7,11 @@ import { nativeProtocol } from '@deepseek-ai/dsh-cua/native-schema' * @returns Native JSON Schema and Swift field/enum tables, including private continuations. */ export function renderNativeProtocol(): { json: string; swift: string } { - const { fields, enums, modelMethods, json } = nativeProtocol() + const { fields, enums, json } = nativeProtocol() const quoted = (values: Iterable) => [...values].sort().map(value => JSON.stringify(value)).join(', ') const swift = [ '// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.', 'enum NativeProtocol {', - ` static let modelMethods: [String] = [${quoted(modelMethods)}]`, ' static let fields: [String: Set] = [', ...[...fields].sort(([a], [b]) => a.localeCompare(b)).map(([method, names]) => ` ${JSON.stringify(method)}: [${quoted(names)}],`), ' ]', diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index 8dc2d9d875..728fcf5899 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -290,6 +290,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "snapshotId": { "type": "string", "minLength": 1 @@ -332,6 +337,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "snapshotId": { "type": "string", "minLength": 1 @@ -415,6 +425,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "snapshotId": { "type": "string", "minLength": 1 @@ -459,6 +474,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "snapshotId": { "type": "string", "minLength": 1 @@ -538,6 +558,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "snapshotId": { "type": "string", "minLength": 1 @@ -693,6 +718,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "snapshotId": { "type": "string", "minLength": 1 @@ -772,6 +802,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "snapshotId": { "type": "string", "minLength": 1 @@ -833,6 +868,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "text": { "type": "string" } @@ -868,6 +908,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "text": { "type": "string" }, @@ -934,6 +979,11 @@ "type": "string", "minLength": 1 }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, "key": { "type": "string", "minLength": 1 diff --git a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py index 7d55cea69c..43aa7d30fb 100644 --- a/packages/experimental/cua-macos/native/tests/capabilities_e2e.py +++ b/packages/experimental/cua-macos/native/tests/capabilities_e2e.py @@ -144,7 +144,6 @@ def main(): call = helper.call capabilities = call("getCapabilities", {}) assert capabilities["permissions"] == {"accessibility": True, "screenRecording": True}, capabilities - assert "launchApp" not in capabilities["methods"], capabilities matches = call("findApps", {"query": str(application), "maxAppMatches": 20}) assert len(matches) == 1 and matches[0]["isRunning"] is False, matches pending = call("getApp", {"selector": str(application)}) diff --git a/packages/experimental/cua-macos/native/tests/e2e.py b/packages/experimental/cua-macos/native/tests/e2e.py index 078ffb62a7..4afa6df75b 100644 --- a/packages/experimental/cua-macos/native/tests/e2e.py +++ b/packages/experimental/cua-macos/native/tests/e2e.py @@ -126,7 +126,6 @@ def main(): time.sleep(0.025) capabilities = call("getCapabilities", {}) - assert capabilities["platform"] == "macos", capabilities assert capabilities["permissions"] == {"accessibility": True, "screenRecording": True}, capabilities apps = call("listApps", {}) assert any(app["pid"] == fixture.pid for app in apps), apps diff --git a/packages/experimental/cua-macos/src/index.ts b/packages/experimental/cua-macos/src/index.ts index 4a22ad7307..9bc9ead89d 100644 --- a/packages/experimental/cua-macos/src/index.ts +++ b/packages/experimental/cua-macos/src/index.ts @@ -3,11 +3,25 @@ import { isAbsolute, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import type { Context } from '@deepseek-ai/cordis' import { CuaConfig, installCua } from '@deepseek-ai/dsh-cua' +import type { CuaStartupCapabilities } from '@deepseek-ai/dsh-cua' import { nativeHostArchitecture, prepareNativeExecutable } from './native-build.ts' import { createSharedNativeChannel } from './shared-native.ts' import { GUIDANCE } from './guidance.ts' export { GUIDANCE } from './guidance.ts' +const CAPABILITIES: CuaStartupCapabilities = { + platform: 'macos', + methods: [ + 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'hover', 'invoke', + 'listActions', 'listApps', 'listWindows', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', + 'selectText', 'setValue', 'show', 'typeText', + ], + appLevelAccessibility: true, + keyboard: ['app'], + browser: false, + coordinateSpace: 'screenshot-pixels', +} + /** Loader plugin identity. */ export const name = 'experimental-cua-macos' /** Services required by the shared CUA executor and the native launcher. */ @@ -27,7 +41,9 @@ export const Config: typeof CuaConfig = CuaConfig export async function apply(ctx: Context, config: Config): Promise { if (config.helperPath !== undefined) { if (!isAbsolute(config.helperPath)) throw new Error('dsh-cua helperPath must be absolute') - installCua(ctx, config, { helperPath: config.helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE }) + installCua(ctx, config, { + helperPath: config.helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE, capabilities: CAPABILITIES, + }) return } const architecture = nativeHostArchitecture(process.platform, process.arch) @@ -44,5 +60,5 @@ export async function apply(ctx: Context, config: Config): Promise { }, 'dsh-cua.native-preparation') const helperPath = await prepared controller.signal.throwIfAborted() - installCua(ctx, config, { helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE }) + installCua(ctx, config, { helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE, capabilities: CAPABILITIES }) } diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 53bf555f27..6333ff69bd 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -162,13 +162,18 @@ it('correlates a native request with its logged tool invocation', async () => { expect(native.request).toHaveBeenCalledExactlyOnceWith('listApps', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1')) }) -it('reports screenshot pixel coordinates while preserving native capabilities', async () => { +it('merges startup capabilities into runtime native fields', async () => { const f = await setup() - const capabilities = { platform: 'macos', coordinateSpace: 'window-points', permissions: { accessibility: true, screenCapture: false }, appLevelAccessibility: true, keyboard: ['app'] } - native.request.mockResolvedValueOnce(capabilities) + const runtime = { permissions: { accessibility: true, screenRecording: false }, sessionLocked: false } + native.request.mockResolvedValueOnce(runtime) const result = await f.execute('getCapabilities') expect(result.isError).toBe(false) - expect(result.value).toEqual({ ...capabilities, coordinateSpace: 'screenshot-pixels' }) + expect(result.value).toEqual({ ...runtime, platform: 'macos', appLevelAccessibility: true, keyboard: ['app'], browser: false, + coordinateSpace: 'screenshot-pixels', methods: [ + 'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'hover', 'invoke', + 'listActions', 'listApps', 'listWindows', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll', + 'selectText', 'setValue', 'show', 'typeText', + ] }) expect(native.request).toHaveBeenCalledExactlyOnceWith('getCapabilities', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1')) }) diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index f31f651311..de9f71bf0a 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -3,7 +3,7 @@ import { Buffer } from 'node:buffer' import type { z } from 'zod' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts' -import type { CuaApi, CuaApp, CuaAppHandle, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts' +import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts' export type * from './types.ts' function requestIssueMessage(issues: readonly z.core.$ZodIssue[], parent: readonly PropertyKey[] = []): string { @@ -56,10 +56,6 @@ function validatedCall(dispatch: CuaDispatch) { } } -async function unsupportedKeyboardInput(): Promise { - throw new Error('Keyboard input is not supported at this scope.') -} - async function emit(sink: CuaObservationSink | undefined, value: object, options?: CuaObservationOptions): Promise { if (options?.emit !== false) await sink?.text(JSON.stringify(value)) } @@ -72,11 +68,17 @@ async function emit(sink: CuaObservationSink | undefined, value: object, options */ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): CuaApi { const call = validatedCall(dispatch) - const objects = createCuaObjects(dispatch, sink) + let capabilities: CuaCapabilities | undefined + async function currentCapabilities(): Promise { + if (capabilities === undefined) capabilities = CapabilitiesResult.parse(await call('getCapabilities', {})) + return capabilities + } + const objects = createCuaObjects(call, sink) const api: CuaApi = { async getCapabilities(options) { const result = CapabilitiesResult.parse(await call('getCapabilities', {})) + capabilities = result await emit(sink, result, options) return result }, @@ -98,82 +100,112 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): return result }, async getApp(selector) { + const request = Request.safeParse({ method: 'getApp', params: { selector } }) + if (!request.success) throw new Error(`getApp ${requestIssueMessage(request.error.issues)}`) + const capabilities = await currentCapabilities() const { appHandle } = AppResult.parse(await call('getApp', { selector })) - const app = objects.createApp(appHandle) - await app.getAccessibilityTree({ emit: false }) + const app = objects.createCuaApp(appHandle, capabilities) + if (capabilities.appLevelAccessibility) await app.getAccessibilityTree({ emit: false }) return app }, } return api } -function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | undefined): { - createApp(appHandle: CuaAppHandle): CuaApp - createWindow(appHandle: CuaAppHandle, windowId: number): CuaWindow +type CuaScopeParams = (params: Record) => Record + +function createCuaObjects(call: ReturnType, sink: CuaObservationSink | undefined): { + createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp } { - const call = validatedCall(dispatch) - const action = async (method: CuaOperation['method'], params: Record) => ActionResult.parse(await call(method, params)) const observed = new Map() - function elementParams(appHandle: CuaAppHandle): Record { + function currentSnapshot(appHandle: CuaAppHandle): string { const current = observed.get(appHandle) if (current === undefined) throw new Error('Read getAccessibilityTree before using this surface') - return { appHandle, snapshotId: current.snapshotId } + return current.snapshotId } function coordinateParams(appHandle: CuaAppHandle, windowId: number): Record { const current = observed.get(appHandle) - return { - appHandle, windowId, - ...current !== undefined && (current.windowId === undefined || current.windowId === windowId) - ? { snapshotId: current.snapshotId } : {}, - } + return current !== undefined && (current.windowId === undefined || current.windowId === windowId) + ? { snapshotId: current.snapshotId } : {} } - function createSurface(appHandle: CuaAppHandle, selectedWindow?: number): CuaSurface { - return { - async show() { - return ShowResult.parse(await call('show', { appHandle, - ...selectedWindow === undefined ? {} : { windowId: selectedWindow }, - })) - }, + function createCuaSurface( + target: { appHandle: CuaAppHandle; windowId?: number }, + paramsFor: CuaScopeParams, + requireAccessibility: () => void, + requireKeyboard: () => void, + ) { + const scopedCall = (method: CuaOperation['method'], params: Record) => + call(method, paramsFor(params)) + const action = async (method: CuaOperation['method'], params: Record) => + ActionResult.parse(await scopedCall(method, params)) + const elementParams = (): Record => ({ snapshotId: currentSnapshot(target.appHandle) }) + const surface: CuaSurface = { + async show() { return ShowResult.parse(await scopedCall('show', {})) }, async getAccessibilityTree(options) { - observed.delete(appHandle) - const state = StateResult.parse(await call('getState', { appHandle, - ...selectedWindow === undefined ? {} : { windowId: selectedWindow }, + requireAccessibility() + observed.delete(target.appHandle) + const state = StateResult.parse(await scopedCall('getState', { ...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus }, ...options?.details === undefined ? {} : { details: options.details }, })) - observed.set(appHandle, { snapshotId: state.snapshotId, windowId: selectedWindow }) + observed.set(target.appHandle, { snapshotId: state.snapshotId, windowId: target.windowId }) if (options?.emit !== false) await sink?.text(state.tree) return state.tree }, async listActions(elementId) { - return ActionNamesResult.parse(await call('listActions', { ...elementParams(appHandle), elementId })) + requireAccessibility() + return ActionNamesResult.parse(await scopedCall('listActions', { ...elementParams(), elementId })) }, async performAction(elementId, name, argument) { - return action('performAction', { ...elementParams(appHandle), elementId, action: name, + requireAccessibility() + return action('performAction', { ...elementParams(), elementId, action: name, ...argument === undefined ? {} : { argument }, }) }, async invoke(elementId) { - return ActionResult.parse(await call('invoke', { ...elementParams(appHandle), elementId })) + requireAccessibility() + return action('invoke', { ...elementParams(), elementId }) + }, + async setValue(elementId, value) { + requireAccessibility() + return action('setValue', { ...elementParams(), elementId, value }) }, - async setValue(elementId, value) { return action('setValue', { ...elementParams(appHandle), elementId, value }) }, async selectText(elementId, text, options) { - return action('selectText', { ...elementParams(appHandle), elementId, text, + requireAccessibility() + return action('selectText', { ...elementParams(), elementId, text, ...options?.prefix === undefined ? {} : { prefix: options.prefix }, ...options?.suffix === undefined ? {} : { suffix: options.suffix }, ...options?.selectionType === undefined ? {} : { selectionType: options.selectionType }, }) }, - typeText: unsupportedKeyboardInput, - paste: unsupportedKeyboardInput, - pressKey: unsupportedKeyboardInput, + async typeText(text) { + requireKeyboard() + return action('typeText', { text }) + }, + async paste(text, options) { + requireKeyboard() + return action('paste', { text, + ...options?.format === undefined ? {} : { format: options.format }, + }) + }, + async pressKey(key) { + requireKeyboard() + return action('pressKey', keyChord(key)) + }, } + return { surface, call: scopedCall, action, elementParams } } - function createWindow(appHandle: CuaAppHandle, windowId: number): CuaWindow { + function createCuaWindow(appHandle: CuaAppHandle, windowId: number, capabilities: CuaCapabilities): CuaWindow { + const base = createCuaSurface({ appHandle, windowId }, + params => ({ ...params, appHandle, windowId }), () => undefined, () => { + if (!capabilities.keyboard.includes('window')) { + throw new Error('Window-scoped keyboard input is not supported on current platform.') + } + }) return { - ...createSurface(appHandle, windowId), windowId, + ...base.surface, windowId, async getScreenshot(options) { - const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId })) + const image = ScreenshotResult.parse(await base.call('screenshot', {})) const screenshot: CuaScreenshot = { bytes: Uint8Array.from(Buffer.from(image.data, 'base64')), mimeType: image.mimeType, @@ -184,37 +216,39 @@ function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | unde if (options?.emit !== false) await sink?.image(screenshot) return screenshot }, - async hover(target, options) { + async hover(targetPoint, options) { const relative = options?.relative === undefined ? {} : { relative: options.relative } - if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...elementParams(appHandle), elementId: target, ...relative })) - const [x, y] = target - return HoverResult.parse(await call('hover', { ...coordinateParams(appHandle, windowId), x, y, ...relative })) + if (typeof targetPoint === 'string') { + return HoverResult.parse(await base.call('hover', { ...base.elementParams(), elementId: targetPoint, ...relative })) + } + const [x, y] = targetPoint + return HoverResult.parse(await base.call('hover', { ...coordinateParams(appHandle, windowId), x, y, ...relative })) }, - async click(target, options) { + async click(targetPoint, options) { const mouseButton = options?.mouseButton ?? 'left' const clickCount = options?.clickCount ?? 1 if (mouseButton !== 'left' && clickCount !== 1) throw new Error('Only left double-click is supported') const modifiers = options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] } const hold = options?.holdMs === undefined ? {} : { holdMs: options.holdMs } const relative = options?.relative === undefined ? {} : { relative: options.relative } - if (typeof target === 'string') { - return action('click', { ...elementParams(appHandle), ...modifiers, ...hold, ...relative, elementId: target, + if (typeof targetPoint === 'string') { + return base.action('click', { ...base.elementParams(), ...modifiers, ...hold, ...relative, elementId: targetPoint, ...mouseButton === 'left' ? {} : { mouseButton }, ...clickCount === 1 ? {} : { clickCount }, }) } - const [x, y] = target - return action('click', { ...coordinateParams(appHandle, windowId), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative }) + const [x, y] = targetPoint + return base.action('click', { ...coordinateParams(appHandle, windowId), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative }) }, async scroll(options) { if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero') - return action('scroll', { + return base.action('scroll', { ...coordinateParams(appHandle, windowId), x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY, ...options.relative === undefined ? {} : { relative: options.relative }, }) }, async drag(from, to, options) { - return action('drag', { ...coordinateParams(appHandle, windowId), + return base.action('drag', { ...coordinateParams(appHandle, windowId), fromX: from[0], fromY: from[1], toX: to[0], toY: to[1], ...options?.relative === undefined ? {} : { relative: options.relative }, ...options?.mouseButton === undefined ? {} : { mouseButton: options.mouseButton }, @@ -223,26 +257,30 @@ function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | unde }, } } - function createApp(appHandle: CuaAppHandle): CuaApp { + function createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp { + const base = createCuaSurface({ appHandle }, params => ({ ...params, appHandle }), () => { + if (!capabilities.appLevelAccessibility) { + throw new Error('Application-level accessibility is not supported on current platform; observe a window instead.') + } + }, () => { + if (!capabilities.keyboard.includes('app')) { + throw new Error('App-scoped keyboard input is not supported on current platform.') + } + }) return { - ...createSurface(appHandle), appHandle, - async activate() { return action('activate', { appHandle }) }, + ...base.surface, appHandle, + async activate() { return base.action('activate', {}) }, async getWindows(options) { - const windows = WindowsResult.parse(await call('listWindows', { appHandle })) + const windows = WindowsResult.parse(await base.call('listWindows', {})) await emit(sink, windows, options) return windows }, async getWindow(windowId): Promise { - const window = createWindow(appHandle, windowId) + const window = createCuaWindow(appHandle, windowId, capabilities) await window.getAccessibilityTree({ emit: false }) return window }, - async typeText(text) { return action('typeText', { appHandle, text }) }, - async paste(text, options) { return action('paste', { appHandle, text, - ...options?.format === undefined ? {} : { format: options.format }, - }) }, - async pressKey(key) { return action('pressKey', { appHandle, ...keyChord(key) }) }, } } - return { createApp, createWindow } + return { createCuaApp } } diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 31c880ac50..c39349bb35 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -16,7 +16,7 @@ import { NativeClient, NativeError, type NativeClientConfig, type NativeChannelF import { CuaPermissionStore } from './permission-store.ts' import { ApplicationAuthorizationResult, installCuaAccess } from './access.ts' import type { CuaApplicationGrant, CuaApplicationId } from './access-types.ts' -import type { CuaAppHandle, CuaOperation } from './types.ts' +import type { CuaAppHandle, CuaOperation, CuaStartupCapabilities } from './types.ts' import { ApplicationIcon, AppResult, NativeScreenshotResult, Request, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts' import { CuaCoordinates } from './coordinates.ts' import { createScope, type Scope } from '@deepseek-ai/dsh-scope' @@ -29,6 +29,7 @@ import type {} from '@deepseek-ai/dsh-ptc-runtime' import type {} from '@deepseek-ai/cordis-plugin-loader' export { CuaAccess } from './access.ts' +export type { CuaStartupCapabilities } from './types.ts' /** Native executable and per-operation resource bounds. */ export interface CuaConfig { @@ -145,11 +146,12 @@ async function screenshotStore(ctx: Context, execution: Readonly) return { attachments, target: (image: ImageAttachmentRef) => llm.imageRequestTarget(provider, model, image) } } -/** Prepared platform executable, transport, and platform-specific operating instructions. */ +/** Prepared platform executable, transport, platform-specific instructions and startup capabilities. */ export interface CuaBackend { helperPath: string createChannel: NativeChannelFactory guidance: string + capabilities: CuaStartupCapabilities } /** @@ -282,7 +284,19 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) return selected } signal.throwIfAborted() - if (method === 'getCapabilities') return { ...z.record(z.string(), z.json()).parse(result), coordinateSpace: 'screenshot-pixels' } + if (method === 'getCapabilities') { + const runtime = z.record(z.string(), z.json()).parse(result) + const capabilities = backend.capabilities + return { + ...runtime, + platform: capabilities.platform, + methods: [...capabilities.methods], + appLevelAccessibility: capabilities.appLevelAccessibility, + keyboard: [...capabilities.keyboard], + browser: capabilities.browser, + coordinateSpace: capabilities.coordinateSpace, + } + } if (method === 'getState') { const state = StateResult.parse(result) execution.coordinates.observe(state.appHandle, state.windows, state.windowId ?? undefined) diff --git a/packages/experimental/cua/src/native-schema.ts b/packages/experimental/cua/src/native-schema.ts index 555ffbe445..5a802f92ca 100644 --- a/packages/experimental/cua/src/native-schema.ts +++ b/packages/experimental/cua/src/native-schema.ts @@ -13,16 +13,14 @@ function variants(schema: Schema): Schema[] { return schema.anyOf ?? schema.oneO /** * Derive native request schemas and admission fields from the shared operation definitions. - * @returns Model operation names, native field and enum tables, and the complete wire schema. + * @returns Native field and enum tables and the complete wire schema. */ export function nativeProtocol(): { fields: Map> enums: Map>> - modelMethods: string[] json: string } { const resources = new Map(Object.entries(NativeResourceParameters)) - const modelMethods = Request.options.map(schema => schema.shape.method.value).sort() const schemas = [...Request.options, ...NativeOnlyOperations] .map(schema => z.toJSONSchema(schema)) const fields = new Map>() @@ -69,7 +67,7 @@ export function nativeProtocol(): { fields.set(method, names) if (values.size) enums.set(method, values) } - return { fields, enums, modelMethods, json: JSON.stringify({ + return { fields, enums, json: JSON.stringify({ $schema: 'https://json-schema.org/draft/2020-12/schema', $comment: 'Generated from the shared CUA operation schema. Do not edit.', title: 'dsh-cua native request', diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 3532cb9e2a..8b3b774b49 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -17,7 +17,7 @@ const window = { ...app, windowId: id } const jsonValue = z.json() export const elementId = z.string().min(1) .describe('Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot.') -const snapshot = { ...app, snapshotId: z.string().min(1), elementId } +const snapshot = { ...app, windowId: id.optional(), snapshotId: z.string().min(1), elementId } const coordinate = { ...window, snapshotId: z.string().min(1).optional(), x: point, y: point } const absoluteCoordinate = { ...coordinate, relative: z.literal(false).optional().describe('Omitted or false uses pixels in the latest screenshot returned by this tool.') } const relativeCoordinate = { ...coordinate, relative: z.literal(true).describe('Use integer thousandths of the window width and height, from 0 to 999.'), x: relativePoint, y: relativePoint } @@ -59,9 +59,9 @@ export const Request = z.discriminatedUnion('method', [ operation('setValue', { ...snapshot, value: jsonValue }), operation('selectText', { ...snapshot, text: z.string().min(1), prefix: z.string().optional(), suffix: z.string().optional(), selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }), - operation('typeText', { ...app, text: z.string() }), - operation('paste', { ...app, text: z.string(), format: z.enum(['text', 'html']).optional() }), - operation('pressKey', { ...app, key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }), + operation('typeText', { ...app, windowId: id.optional(), text: z.string() }), + operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }), + operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }), z.object({ method: z.literal('scroll'), params: z.union([ z.object({ ...absoluteCoordinate, deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(), diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 9ebeab936b..9a5a3372c9 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -45,6 +45,11 @@ export interface CuaAxOptions extends CuaObservationOptions { } /** Reported permissions and operations; null sessionLocked means no usable console session was verified. */ export type CuaCapabilities = z.output +/** Provider capabilities fixed when the provider is installed; runtime fields come from getCapabilities. */ +export type CuaStartupCapabilities = Pick< + CuaCapabilities, + 'platform' | 'methods' | 'appLevelAccessibility' | 'keyboard' | 'browser' | 'coordinateSpace' +> /** App discovery result accepted by getApp. */ export type CuaAppInfo = Pick, 'id' | 'displayName' | 'isRunning' | 'pid'> /** An installed application match; running matches include a process identifier. */ diff --git a/packages/experimental/cua/tests/agent-scopes.spec.ts b/packages/experimental/cua/tests/agent-scopes.spec.ts index 0c25e1864b..ac7430e397 100644 --- a/packages/experimental/cua/tests/agent-scopes.spec.ts +++ b/packages/experimental/cua/tests/agent-scopes.spec.ts @@ -19,6 +19,10 @@ const provider = { helperPath: resolve('unused-cua-helper'), createChannel() { throw new Error('Scope tests must not start a native helper') }, guidance: 'Fixture CUA guidance', + capabilities: { + platform: 'macos', methods: ['getCapabilities', 'getApp', 'getState'], appLevelAccessibility: true, + keyboard: ['app'], browser: false, coordinateSpace: 'screenshot-pixels', + }, }) }, } diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index ebd96a0f5f..50b3d306de 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -4,7 +4,7 @@ import { describe, expect, expectTypeOf, it, vi } from 'vitest' import { brandString } from '@deepseek-ai/dsh-brand' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { createCuaApi } from '../src/api.ts' -import type { CuaApp, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts' +import type { CuaApp, CuaCapabilities, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts' /** Element ids are opaque strings; these fixtures use the macOS decimal form. */ const element = (id: string): CuaElementId => brandString(id) @@ -13,7 +13,7 @@ const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG const bounds = { x: -640, y: 100, width: 640, height: 480 } const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', running: true } const delivered = { dispatched: true } -const capabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false } +const capabilities: CuaCapabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false } function windowInfo(windowId: number) { return { windowId, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null } @@ -21,11 +21,12 @@ function windowInfo(windowId: number) { function fixture(withSink = true) { let windows = [windowInfo(23)] + let currentCapabilities: CuaCapabilities = capabilities let generation = 0 const sink = { text: vi.fn<(value: string) => void>(), image: vi.fn<(screenshot: CuaScreenshot) => void>() } const dispatch = vi.fn(async ({ method, params }: CuaOperation): Promise => { switch (method) { - case 'getCapabilities': return capabilities + case 'getCapabilities': return currentCapabilities case 'listApps': return [selected, { pid: 18, name: 'Utility' }] case 'getApp': return selected case 'listWindows': return windows @@ -45,6 +46,7 @@ function fixture(withSink = true) { dispatch, sink, api: createCuaApi(dispatch, withSink ? sink : undefined), setWindows: (ids: number[]) => { windows = ids.map(windowInfo) }, setWindowRecords: (records: ReturnType[]) => { windows = records }, + setCapabilities: (value: CuaCapabilities) => { currentCapabilities = value }, } } @@ -163,7 +165,7 @@ describe('hover across observations', () => { await window.getScreenshot({ emit: false }) expect(await window.hover([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false }) expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([ - { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0' } }, + { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: 23 } }, { method: 'getState', params: { appHandle: 'app-17', windowId: 23 } }, { method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } }, { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 23, x: 30, y: 40 } }, @@ -199,7 +201,7 @@ describe('pointer buttons and modifiers', () => { const window = await app.getWindow(23) await window.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] }) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { - appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', mouseButton: 'middle', modifiers: ['shift'], + appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: 23, mouseButton: 'middle', modifiers: ['shift'], } }) await window.click([10, 20], { modifiers: ['command'] }) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { @@ -322,18 +324,18 @@ describe('application and window observations', () => { await second.click(element('3')) expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([ { method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '1', value: 'First document' } }, - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2' } }, - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3' } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2', windowId: 23 } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3', windowId: 24 } }, ]) expect(first.windowId).toBe(23) expect(second.windowId).toBe(24) expect(f.sink.text.mock.calls.map(([text]) => text)).toEqual([]) await expect(second.getAccessibilityTree({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4') await second.click(element('3')) - expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3' } }) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3', windowId: 24 } }) await app.getAccessibilityTree({ emit: false }) await first.invoke(element('4')) - expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4' } }) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4', windowId: 23 } }) expect(f.sink.text).not.toHaveBeenCalled() }) @@ -476,6 +478,69 @@ describe('application and window observations', () => { await expect(app.getWindows()).resolves.toEqual([observed]) }) + it('skips app-level accessibility while keeping window observation available', async () => { + const f = fixture() + f.setCapabilities({ ...capabilities, appLevelAccessibility: false }) + const app = await f.api.getApp('Editor') + expect(f.dispatch.mock.calls.map(([call]) => call.method)).toEqual(['getCapabilities', 'getApp']) + await expect(app.getAccessibilityTree()).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.') + await expect(app.listActions(element('0'))).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.') + await expect(app.performAction(element('0'), 'AXPress')).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.') + await expect(app.invoke(element('0'))).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.') + await expect(app.setValue(element('0'), 'value')).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.') + await expect(app.selectText(element('0'), 'text')).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.') + expect(f.dispatch).toHaveBeenCalledTimes(2) + f.dispatch.mockClear() + await expect(app.typeText('keyboard without accessibility')).resolves.toEqual(delivered) + expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([ + { method: 'typeText', params: { appHandle: 'app-17', text: 'keyboard without accessibility' } }, + ]) + const window = await app.getWindow(23) + await expect(window.getAccessibilityTree({ emit: false })).resolves.toBe('Tree 2') + await expect(app.getAccessibilityTree()).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.') + await expect(window.invoke(element('0'))).resolves.toEqual(delivered) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { + appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', elementId: '0', + } }) + }) + + it('honors window keyboard support and keeps app keyboard available', async () => { + const f = fixture() + f.setCapabilities({ ...capabilities, keyboard: ['app', 'window'] }) + const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) + f.dispatch.mockClear() + await window.typeText('hello') + await window.paste('bold', { format: 'html' }) + await window.pressKey('cmd+a') + await app.typeText('app text') + await app.paste('app paste') + await app.pressKey('shift+Tab') + expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([ + { method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'hello' } }, + { method: 'paste', params: { appHandle: 'app-17', windowId: 23, text: 'bold', format: 'html' } }, + { method: 'pressKey', params: { appHandle: 'app-17', windowId: 23, key: 'a', modifiers: ['command'] } }, + { method: 'typeText', params: { appHandle: 'app-17', text: 'app text' } }, + { method: 'paste', params: { appHandle: 'app-17', text: 'app paste' } }, + { method: 'pressKey', params: { appHandle: 'app-17', key: 'Tab', modifiers: ['shift'] } }, + ]) + }) + + it('rejects app keyboard when only window scope is supported', async () => { + const f = fixture() + f.setCapabilities({ ...capabilities, keyboard: ['window'] }) + const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) + f.dispatch.mockClear() + await expect(app.typeText('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') + await expect(app.paste('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') + await expect(app.pressKey('Return')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') + await expect(app.pressKey('unknown+a')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') + expect(f.dispatch).not.toHaveBeenCalled() + await expect(window.typeText('window text')).resolves.toEqual(delivered) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'window text' } }) + }) + it('accepts coordinate input without a current snapshot', async () => { const f = fixture() const app = await f.api.getApp('Editor') @@ -686,8 +751,9 @@ describe('input actions', () => { () => window.typeText('hello'), () => window.paste('hello'), () => window.pressKey('Return'), + () => window.pressKey('unknown+a'), ]) { - await expect(run()).rejects.toThrow('Keyboard input is not supported at this scope.') + await expect(run()).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.') } expect(f.dispatch).not.toHaveBeenCalled() }) @@ -699,8 +765,8 @@ describe('input actions', () => { await window.click(element('12'), { mouseButton: 'right' }) await window.click(element('13'), { clickCount: 2 }) expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([ - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', mouseButton: 'right' } }, - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', clickCount: 2 } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', windowId: 23, mouseButton: 'right' } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', windowId: 23, clickCount: 2 } }, ]) }) @@ -711,15 +777,15 @@ describe('input actions', () => { await window.click(element('42.12345')) await window.hover(element('42.12345')) expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([ - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } }, - { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: 23 } }, + { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: 23 } }, ]) }) it('does not activate on selection and dispatches only an explicit activation request', async () => { const f = fixture() const app = await f.api.getApp('Editor') - expect(f.dispatch.mock.calls.map(([call]) => call.method)).toEqual(['getApp', 'getState']) + expect(f.dispatch.mock.calls.map(([call]) => call.method)).toEqual(['getCapabilities', 'getApp', 'getState']) await app.activate() expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'activate', params: { appHandle: 'app-17' } }) }) @@ -731,10 +797,10 @@ describe('input actions', () => { const failure = new Error('Application was restarted; select it again') f.dispatch.mockRejectedValueOnce(failure) await expect(app.typeText('hello')).rejects.toBe(failure) - expect(f.dispatch).toHaveBeenCalledTimes(4) + expect(f.dispatch).toHaveBeenCalledTimes(5) f.dispatch.mockResolvedValueOnce({ dispatched: 'yes' }) await expect(window.click(element('0'))).rejects.toThrow() - expect(f.dispatch).toHaveBeenCalledTimes(5) + expect(f.dispatch).toHaveBeenCalledTimes(6) }) it('rejects invalid selection inputs before dispatch', async () => { diff --git a/packages/experimental/cua/tests/bridge.spec.ts b/packages/experimental/cua/tests/bridge.spec.ts index 36739602db..14a36baaba 100644 --- a/packages/experimental/cua/tests/bridge.spec.ts +++ b/packages/experimental/cua/tests/bridge.spec.ts @@ -5,12 +5,17 @@ import { brandString } from '@deepseek-ai/dsh-brand' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { connectCua } from '../src/bridge.ts' import type { CuaToolBinding } from '../src/bridge.ts' -import type { CuaElementId, CuaOperation } from '../src/types.ts' +import type { CuaCapabilities, CuaElementId, CuaOperation } from '../src/types.ts' /** Element ids are opaque strings; these fixtures use the macOS decimal form. */ const element = (id: string): CuaElementId => brandString(id) const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC' +const capabilities: CuaCapabilities = { + platform: 'macos', permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: true, + keyboard: ['app'], sessionLocked: false, methods: ['getCapabilities', 'getApp', 'getState', 'screenshot'], + coordinateSpace: 'screenshot-pixels', browser: false, +} const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', running: true } const bounds = { x: 0, y: 0, width: 640, height: 480 } const snapshot = { @@ -20,6 +25,7 @@ const snapshot = { it('uses the current tool binding when a retained app is called by another execution', async () => { const first = vi.fn(async (operation: CuaOperation): Promise => { + if (operation.method === 'getCapabilities') return capabilities if (operation.method === 'getApp') return selected if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null } return snapshot @@ -33,8 +39,8 @@ it('uses the current tool binding when a retained app is called by another execu current = second await expect(window.click(element('0'))).resolves.toEqual({ dispatched: true }) - expect(first).toHaveBeenCalledTimes(3) - expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } }) + expect(first).toHaveBeenCalledTimes(4) + expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', windowId: 23 } }) expect(text).not.toHaveBeenCalled() current = first await app.getAccessibilityTree() @@ -61,6 +67,7 @@ it.each([null, 'text only', { content: [{ type: 'text', text: '{}' }] }])('rejec it('returns screenshot bytes and metadata with one image-bearing tool call and no image text output', async () => { const tool = vi.fn(async (operation: CuaOperation): Promise => { + if (operation.method === 'getCapabilities') return capabilities if (operation.method === 'getApp') return selected if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null } return { @@ -73,6 +80,6 @@ it('returns screenshot bytes and metadata with one image-bearing tool call and n text.mockClear() expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 }) - expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getApp', 'getState', 'getState', 'screenshot']) + expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getCapabilities', 'getApp', 'getState', 'getState', 'screenshot']) expect(text).not.toHaveBeenCalled() }) diff --git a/packages/experimental/cua/tests/native-schema.spec.ts b/packages/experimental/cua/tests/native-schema.spec.ts index c5c157991a..358b87021f 100644 --- a/packages/experimental/cua/tests/native-schema.spec.ts +++ b/packages/experimental/cua/tests/native-schema.spec.ts @@ -33,7 +33,7 @@ it('retains optional resource fields and tuple-valued parameters without inventi }) it('admits only locator fields for default invocation', () => { - expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId'])) + expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId', 'windowId'])) }) it('accepts a parameter-free object schema without a properties dictionary', () => {