From bfd5e9fccc3cf540d2380a8db11db7b392e7d581 Mon Sep 17 00:00:00 2001 From: Huo Yaoyuan Date: Wed, 4 Nov 2026 13:22:42 +0800 Subject: [PATCH] refactor(cua): separate window operations from accessibility surfaces --- .../native/Sources/CuaNative/AXWorker.swift | 1 + .../tests/loader-composition.spec.ts | 6 +- .../tests/persistent-bridge.built.e2e.ts | 2 +- .../tests/provider-lifecycle.spec.ts | 2 +- packages/experimental/cua/src/api.ts | 239 ++++++------ packages/experimental/cua/src/bridge.ts | 2 +- packages/experimental/cua/src/protocol.ts | 2 + packages/experimental/cua/src/types.ts | 90 ++--- packages/experimental/cua/src/window-api.ts | 24 +- packages/experimental/cua/src/window-types.ts | 6 +- packages/experimental/cua/tests/api.spec.ts | 353 ++++++++++-------- .../experimental/cua/tests/bridge.spec.ts | 20 +- .../experimental/cua/tests/protocol.spec.ts | 12 +- .../experimental/cua/tests/window-api.spec.ts | 13 +- .../cua-native/workspace/native-helper.mjs | 2 +- 15 files changed, 428 insertions(+), 346 deletions(-) diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index ce3f33eeef..9da63aa413 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -193,6 +193,7 @@ actor AXWorker { if method == "getCapabilities" { return .result(.object(["platform": .string("macos"), "permissions": .object([ "accessibility": .bool(AXIsProcessTrusted()), "screenRecording": .bool(CGPreflightScreenCaptureAccess())]), + "appLevelAccessibility": .bool(true), "sessionLocked": locked.map(JSONValue.bool) ?? .null, "methods": .array(NativeProtocol.modelMethods.map(JSONValue.string)), "coordinateSpace": .string("window-points"), "browser": .bool(false)])) } diff --git a/packages/experimental/cua-macos/tests/loader-composition.spec.ts b/packages/experimental/cua-macos/tests/loader-composition.spec.ts index 834c2845a0..717529d921 100644 --- a/packages/experimental/cua-macos/tests/loader-composition.spec.ts +++ b/packages/experimental/cua-macos/tests/loader-composition.spec.ts @@ -173,7 +173,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained }) const retainedApp = await objectApi(agent).getApp('test.fixture') const retainedWindow = await retainedApp.getWindow(7) - expect(await retainedWindow.getAXState({ emit: false })).toContain('Count: 1') + expect(await retainedWindow.getAccessibilityTree({ emit: false })).toContain('Count: 1') const peer = await context.agents.create({ sessionId: SessionId('native-loader-peer'), agentOptions: { provider: 'native-fixture', model: 'vision' } }) const peerApp = await objectApi(peer.agent).getApp('test.fixture') const peerWindow = await peerApp.getWindow(7) @@ -182,7 +182,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained expect(screenshot).toMatchObject({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 7 }) expect(Buffer.from(screenshot.bytes).toString('base64')).toBe(png) await peerWindow.click(element('0')) - expect(await retainedWindow.getAXState({ emit: false })).toContain('Count: 2') + expect(await retainedWindow.getAccessibilityTree({ emit: false })).toContain('Count: 2') const sharedEvents = await readEvents() expect(sharedEvents.filter(event => event.event === 'start')).toHaveLength(1) expect(sharedEvents.filter(event => event.event === 'connect')).toHaveLength(2) @@ -191,7 +191,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained expect((await readEvents()).filter(event => event.event === 'exit')).toHaveLength(0) expect((await readEvents()).filter(event => event.event === 'disconnect')).toHaveLength(1) await peerWindow.click(element('0')) - expect(await peerWindow.getAXState({ emit: false })).toContain('Count: 3') + expect(await peerWindow.getAccessibilityTree({ emit: false })).toContain('Count: 3') await peer.dispose() expect((await readEvents()).filter(event => event.event === 'exit')).toHaveLength(1) diff --git a/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts b/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts index 95078c1278..39761b0820 100644 --- a/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts +++ b/packages/experimental/cua-macos/tests/persistent-bridge.built.e2e.ts @@ -84,7 +84,7 @@ it.skipIf(process.platform === 'win32' || !existsSync(bridge) || !existsSync(boo expect(invalid.error?.message).not.toContain('ZodError') expect(invalid.error?.message).not.toContain('"origin"') expect(calls.filter(call => call.phase === 'invalid')).toEqual([]) - const second = await run('await win.getAXState({emit:false}); await win.click("0"); await win.getAXState({emit:false});', 'second') + const second = await run('await win.getAccessibilityTree({emit:false}); await win.click("0"); await win.getAccessibilityTree({emit:false});', 'second') expect(second.error).toBeUndefined() expect(second.value).toContain('Count: 2') expect(calls.filter(call => call.phase === 'second').map(call => call.method)).toEqual(['getState', 'click', 'getState']) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 1796deb0e4..9aeb1bf7b0 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -164,7 +164,7 @@ it('correlates a native request with its logged tool invocation', async () => { it('reports screenshot pixel coordinates while preserving native capabilities', async () => { const f = await setup() - const capabilities = { platform: 'macos', coordinateSpace: 'window-points', permissions: { accessibility: true, screenCapture: false } } + const capabilities = { platform: 'macos', coordinateSpace: 'window-points', permissions: { accessibility: true, screenCapture: false }, appLevelAccessibility: true } native.request.mockResolvedValueOnce(capabilities) const result = await f.execute('getCapabilities') expect(result.isError).toBe(false) diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index 7966458742..5e34e8a445 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -68,7 +68,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): async function emit(value: object, options?: CuaObservationOptions): Promise { if (options?.emit !== false) await sink?.text(JSON.stringify(value)) } - + const objects = createCuaObjects(dispatch, sink) const api: CuaApi = { async getCapabilities(options) { @@ -96,7 +96,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): async getApp(selector) { const { appHandle, pid } = AppResult.parse(await call('getApp', { selector })) const app: CuaApp = { - ...createCuaSurface(dispatch, sink, appHandle, pid), pid, appHandle, + ...objects.createSurface(appHandle, pid), pid, appHandle, async activate() { return action('activate', { appHandle }) }, async getWindows(options) { const windows = WindowsResult.parse(await call('listWindows', { appHandle })) @@ -105,8 +105,8 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): return windows }, async getWindow(windowId): Promise { - const window = { ...createCuaSurface(dispatch, sink, appHandle, pid, windowId), windowId } - await window.getAXState({ emit: false }) + const window = objects.createWindow(appHandle, pid, windowId) + await window.getAccessibilityTree({ emit: false }) return window }, async typeText(text) { return action('typeText', { appHandle, text }) }, @@ -115,7 +115,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): }) }, async pressKey(key) { return action('pressKey', { appHandle, ...keyChord(key) }) }, } - await app.getAXState({ emit: false }) + await app.getAccessibilityTree({ emit: false }) return app }, } @@ -123,127 +123,128 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): } /** - * Bind one observed surface to its authorized dispatcher. + * Create application and window objects sharing their latest observations. * @param dispatch Logged operation dispatcher. * @param sink Optional observation output. - * @param appHandle Native authorization handle. - * @param pid Observed process lifetime owner. - * @param selectedWindow Explicit window selection, or the macOS application scope. - * @returns Observation and input methods retaining only the latest AX state. + * @returns Surface and window constructors with observation state owned by this factory. */ -export function createCuaSurface(dispatch: CuaDispatch, sink: CuaObservationSink | undefined, - appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface { +export function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | undefined): { + createSurface(appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface + createWindow(appHandle: CuaAppHandle, pid: number, windowId: number): CuaWindow +} { const call = validatedCall(dispatch) const action = async (method: CuaOperation['method'], params: Record) => ActionResult.parse(await call(method, params)) - let observed: ReturnType | undefined - function snapshotParams(): Record { - if (!observed) throw new Error('Read getAXState before using this surface') - return { appHandle, snapshotId: observed.snapshotId } + const observed = new Map>() + function elementParams(appHandle: CuaAppHandle): Record { + const current = observed.get(appHandle) + if (current === undefined) throw new Error('Read getAccessibilityTree before using this surface') + return { appHandle, snapshotId: current.snapshotId } } - function windowId(): number { - if (selectedWindow !== undefined) return selectedWindow - const windows = observed?.windows.filter(window => window.isOnScreen && window.bounds.width > 0 && window.bounds.height > 0) - const focused = windows?.find(window => window.windowId === observed?.focusedWindowId) - if (focused) return focused.windowId - const first = windows?.[0] - if (!first || windows.length !== 1) throw new Error('Select a window with app.getWindows() and app.getWindow(windowId) before this operation') - return first.windowId + function coordinateParams(appHandle: CuaAppHandle, windowId: number): Record { + const current = observed.get(appHandle) + return { + appHandle, windowId, + ...current !== undefined && (current.windowId === null || current.windowId === windowId) + ? { snapshotId: current.snapshotId } : {}, + } } - const result: CuaSurface = { - async show() { - return ShowResult.parse(await call('show', { appHandle, - ...selectedWindow === undefined ? {} : { windowId: selectedWindow }, - })) - }, - async getAXState(options) { - observed = undefined - const state = StateResult.parse(await call('getState', { appHandle, - ...selectedWindow === undefined ? {} : { windowId: selectedWindow }, - ...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus }, - ...options?.details === undefined ? {} : { details: options.details }, - })) - if (state.appHandle !== appHandle || state.pid !== pid || state.windowId !== (selectedWindow ?? null)) { - throw new Error('Native observation belongs to another application or window') - } - if (state.windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application') - observed = state - if (options?.emit !== false) await sink?.text(state.tree) - return state.tree - }, - async getScreenshot(options) { - if (!observed) await result.getAXState({ emit: false }) - const target = windowId() - const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId: target })) - if (image.windowId !== target) throw new Error('Native screenshot belongs to another window') - const screenshot: CuaScreenshot = { - bytes: Uint8Array.from(Buffer.from(image.data, 'base64')), - mimeType: image.mimeType, - pixelWidth: image.pixelWidth, - pixelHeight: image.pixelHeight, - windowId: image.windowId, - } - if (options?.emit !== false) await sink?.image(screenshot) - return screenshot - }, - async getAXStateAndScreenshot(options) { - const state = await result.getAXState(options) - return { state, screenshot: await result.getScreenshot(options) } - }, - async listActions(elementId) { - return ActionNamesResult.parse(await call('listActions', { ...snapshotParams(), elementId })) - }, - async performAction(elementId, name, argument) { - return action('performAction', { ...snapshotParams(), elementId, action: name, - ...argument === undefined ? {} : { argument }, - }) - }, - async invoke(elementId) { - return ActionResult.parse(await call('invoke', { ...snapshotParams(), elementId })) - }, - async hover(target, options) { - const relative = options?.relative === undefined ? {} : { relative: options.relative } - if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...snapshotParams(), elementId: target, ...relative })) - const [x, y] = target - return HoverResult.parse(await call('hover', { ...snapshotParams(), windowId: windowId(), x, y, ...relative })) - }, - async click(target, options) { - const mouseButton = options?.mouseButton ?? 'left' - const clickCount = options?.clickCount ?? 1 - if (mouseButton !== 'left' && clickCount !== 1) throw new Error('Only left double-click is supported') - const modifiers = options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] } - const hold = options?.holdMs === undefined ? {} : { holdMs: options.holdMs } - const relative = options?.relative === undefined ? {} : { relative: options.relative } - if (typeof target === 'string') { - return action('click', { ...snapshotParams(), ...modifiers, ...hold, ...relative, elementId: target, - ...mouseButton === 'left' ? {} : { mouseButton }, ...clickCount === 1 ? {} : { clickCount }, + function createSurface(appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface { + return { + async show() { + return ShowResult.parse(await call('show', { appHandle, + ...selectedWindow === undefined ? {} : { windowId: selectedWindow }, + })) + }, + async getAccessibilityTree(options) { + observed.delete(appHandle) + const state = StateResult.parse(await call('getState', { appHandle, + ...selectedWindow === undefined ? {} : { windowId: selectedWindow }, + ...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus }, + ...options?.details === undefined ? {} : { details: options.details }, + })) + if (state.appHandle !== appHandle || state.pid !== pid || state.windowId !== (selectedWindow ?? null)) { + throw new Error('Native observation belongs to another application or window') + } + if (state.windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application') + observed.set(appHandle, state) + if (options?.emit !== false) await sink?.text(state.tree) + return state.tree + }, + async listActions(elementId) { + return ActionNamesResult.parse(await call('listActions', { ...elementParams(appHandle), elementId })) + }, + async performAction(elementId, name, argument) { + return action('performAction', { ...elementParams(appHandle), elementId, action: name, + ...argument === undefined ? {} : { argument }, }) - } - const [x, y] = target - return action('click', { ...snapshotParams(), windowId: windowId(), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative }) - }, - async setValue(elementId, value) { return action('setValue', { ...snapshotParams(), elementId, value }) }, - async selectText(elementId, text, options) { - return action('selectText', { ...snapshotParams(), elementId, text, - ...options?.prefix === undefined ? {} : { prefix: options.prefix }, - ...options?.suffix === undefined ? {} : { suffix: options.suffix }, - ...options?.selectionType === undefined ? {} : { selectionType: options.selectionType }, - }) - }, - async scroll(options) { - if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero') - return action('scroll', { - ...snapshotParams(), windowId: windowId(), - x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY, - ...options.relative === undefined ? {} : { relative: options.relative }, - }) - }, - async drag(from, to, options) { - return action('drag', { ...snapshotParams(), windowId: windowId(), fromX: from[0], fromY: from[1], toX: to[0], toY: to[1], - ...options?.relative === undefined ? {} : { relative: options.relative }, - ...options?.mouseButton === undefined ? {} : { mouseButton: options.mouseButton }, - ...options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] }, - }) - }, + }, + async invoke(elementId) { + return ActionResult.parse(await call('invoke', { ...elementParams(appHandle), elementId })) + }, + async setValue(elementId, value) { return action('setValue', { ...elementParams(appHandle), elementId, value }) }, + async selectText(elementId, text, options) { + return action('selectText', { ...elementParams(appHandle), elementId, text, + ...options?.prefix === undefined ? {} : { prefix: options.prefix }, + ...options?.suffix === undefined ? {} : { suffix: options.suffix }, + ...options?.selectionType === undefined ? {} : { selectionType: options.selectionType }, + }) + }, + } } - return result + function createWindow(appHandle: CuaAppHandle, pid: number, windowId: number): CuaWindow { + return { + ...createSurface(appHandle, pid, windowId), windowId, + async getScreenshot(options) { + const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId })) + if (image.windowId !== windowId) throw new Error('Native screenshot belongs to another window') + const screenshot: CuaScreenshot = { + bytes: Uint8Array.from(Buffer.from(image.data, 'base64')), + mimeType: image.mimeType, + pixelWidth: image.pixelWidth, + pixelHeight: image.pixelHeight, + windowId: image.windowId, + } + if (options?.emit !== false) await sink?.image(screenshot) + return screenshot + }, + async hover(target, options) { + const relative = options?.relative === undefined ? {} : { relative: options.relative } + if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...elementParams(appHandle), elementId: target, ...relative })) + const [x, y] = target + return HoverResult.parse(await call('hover', { ...coordinateParams(appHandle, windowId), x, y, ...relative })) + }, + async click(target, options) { + const mouseButton = options?.mouseButton ?? 'left' + const clickCount = options?.clickCount ?? 1 + if (mouseButton !== 'left' && clickCount !== 1) throw new Error('Only left double-click is supported') + const modifiers = options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] } + const hold = options?.holdMs === undefined ? {} : { holdMs: options.holdMs } + const relative = options?.relative === undefined ? {} : { relative: options.relative } + if (typeof target === 'string') { + return action('click', { ...elementParams(appHandle), ...modifiers, ...hold, ...relative, elementId: target, + ...mouseButton === 'left' ? {} : { mouseButton }, ...clickCount === 1 ? {} : { clickCount }, + }) + } + const [x, y] = target + return action('click', { ...coordinateParams(appHandle, windowId), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative }) + }, + async scroll(options) { + if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero') + return action('scroll', { + ...coordinateParams(appHandle, windowId), + x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY, + ...options.relative === undefined ? {} : { relative: options.relative }, + }) + }, + async drag(from, to, options) { + return action('drag', { ...coordinateParams(appHandle, windowId), + fromX: from[0], fromY: from[1], toX: to[0], toY: to[1], + ...options?.relative === undefined ? {} : { relative: options.relative }, + ...options?.mouseButton === undefined ? {} : { mouseButton: options.mouseButton }, + ...options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] }, + }) + }, + } + } + return { createSurface, createWindow } } diff --git a/packages/experimental/cua/src/bridge.ts b/packages/experimental/cua/src/bridge.ts index ed40ecf7a7..407423d11f 100644 --- a/packages/experimental/cua/src/bridge.ts +++ b/packages/experimental/cua/src/bridge.ts @@ -12,7 +12,7 @@ export type CuaToolBinding = (operation: CuaOperation) => Promise * Connect the object API without owning an evaluator or its lifetime. * @param getTool - Resolves the current execution's tools.cua binding on every operation; do not capture a completed execution's binding. * @param onText - Optional text output captured by the caller's execution environment. - * @returns App objects retaining tool approval, logging, image delivery and screenshot metadata. Rejections propagate without retries. + * @returns Application and window objects using the current tool binding. Rejections propagate without retries. */ export function connectCua(getTool: () => CuaToolBinding, onText?: (text: string) => void | Promise): CuaApi { return createCuaApi(operation => getTool()(operation), { diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 22169f6520..2e650bf4b6 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -131,6 +131,8 @@ export const ScreenshotResult = NativeScreenshotResult.extend({ /** Permissions are reported without opening system permission dialogs. */ export const CapabilitiesResult = z.object({ platform: z.string(), permissions: z.object({ accessibility: z.boolean(), screenRecording: z.boolean() }), + /** Whether the provider supports application-level accessibility observations. */ + appLevelAccessibility: z.boolean(), sessionLocked: z.boolean().nullable(), methods: z.array(z.string()), coordinateSpace: z.literal('screenshot-pixels'), browser: z.literal(false), }) diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 34d442aec7..754ff9dc94 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -34,7 +34,7 @@ export interface CuaObservationSink { } /** Observations emit by default. Returned values are unaffected by this option. */ export interface CuaObservationOptions { emit?: boolean } -/** Full observations of the requested content; each read replaces this surface's current snapshot. */ +/** Full observations of the requested content; each read replaces the application's current snapshot. */ export interface CuaAxOptions extends CuaObservationOptions { /** Reserved compatibility option; observations are never diffs. */ disableDiffing?: boolean @@ -85,12 +85,8 @@ export type CuaPasteOptions = Pick, 'format'> export type CuaSelectTextOptions = Pick, 'prefix' | 'suffix' | 'selectionType'> /** Positive deltas reveal content above/left; each axis is an integer from -10 to 10. */ export type CuaScrollOptions = Pick, 'x' | 'y' | 'deltaLinesX' | 'deltaLinesY'> & CuaCoordinateOptions -/** An observed AX tree and independently captured image; capture is not atomic. */ -export interface CuaObservation { state: string; screenshot: CuaScreenshot } /** - * App-wide or explicitly scoped window observations and actions. - * Capture and pointer delivery reject known AX/display frame conflicts with WINDOW_CHANGED. - * Coordinate input also rejects resizing since the latest successful AX read or screenshot. + * Accessibility observations and element operations shared by an application and its windows. * Advertised AX actions remain available; failed actions are not retried through pointer input. */ export interface CuaSurface { @@ -100,32 +96,21 @@ export interface CuaSurface { */ show(): Promise /** - * Read fresh AX text, with menu contents collapsed by default. + * Read the full accessibility tree, with menu contents collapsed by default. + * The returned tree becomes the application's current snapshot; use element ids from its latest successful read. * @param options - emission, menu expansion and field detail options. - * @returns AX text with element ids. + * @returns Full accessibility tree text with element ids, subject to the provider's traversal limits. */ - getAXState(options?: CuaAxOptions): Promise - /** - * Capture the selected window. - * @param options - emission options. - * @returns encoded model-request image with its format, pixel dimensions and window; use mimeType when saving or declaring the image. - */ - getScreenshot(options?: CuaObservationOptions): Promise - /** - * Read AX then capture. - * @param options - observation options. - * @returns both non-atomic observations. - */ - getAXStateAndScreenshot(options?: CuaAxOptions): Promise + getAccessibilityTree(options?: CuaAxOptions): Promise /** * List the actions currently advertised by an element, including attribute-setting operations. - * @param elementId - Element id from this surface's latest observation. + * @param elementId - Element id from the application's current snapshot. * @returns Provider-defined action names; empty when the element advertises none. */ listActions(elementId: CuaElementId): Promise /** * Perform an element action, including attribute-setting operations. - * @param elementId - Element id from this surface's latest observation. + * @param elementId - Element id from the application's current snapshot. * @param action - Exact action name from listActions. * @param argument - Direct action argument; omit or pass null for a no-argument action, or pass the value itself for attribute setting. * @returns Whether the provider successfully executed the requested operation. @@ -134,10 +119,39 @@ export interface CuaSurface { /** * Invoke the element's default action. * If the element's type does not support a default action, the result's dispatched field is false. - * @param elementId - Element id from this surface's latest observation. + * @param elementId - Element id from the application's current snapshot. * @returns Whether the provider successfully executed the requested operation. */ invoke(elementId: CuaElementId): Promise + /** + * Set an observed control value. + * @param elementId - Element id from the application's current snapshot. + * @param value - Replacement value as JSON; supported values are defined by the platform provider. + * @returns dispatch evidence. + */ + setValue(elementId: CuaElementId, value: JsonValue): Promise + /** + * Select one literal text occurrence through writable AXSelectedTextRange and verify its range. + * @param elementId - Element id from the application's current snapshot. + * @param text - Nonempty literal text; multiple matches require adjacent context. + * @param options - Adjacent prefix/suffix and selection or cursor placement. + * @returns Dispatch evidence; rejects unsupported, ambiguous or changed text without keyboard fallback. + */ + selectText(elementId: CuaElementId, text: string, options?: CuaSelectTextOptions): Promise +} +/** + * An explicit window with screenshot and pointer input operations. + * Capture and pointer delivery reject known AX/display frame conflicts with WINDOW_CHANGED. + * Coordinate input also rejects resizing since the latest successful AX read or screenshot. + */ +export interface CuaWindow extends CuaSurface { + readonly windowId: number + /** + * Capture the selected window. + * @param options - emission options. + * @returns encoded model-request image with its format, pixel dimensions and window; use mimeType when saving or declaring the image. + */ + getScreenshot(options?: CuaObservationOptions): Promise /** * Reveal advertised AX alternate UI, otherwise move the system cursor without pressing a button. * The system-pointer route requires the app to be foreground and the point unobscured; never activates or raises it. @@ -156,21 +170,6 @@ export interface CuaSurface { * @returns dispatch evidence. */ click(target: CuaElementId | readonly [number, number], options?: CuaClickOptions): Promise - /** - * Set an observed control value. - * @param elementId - element id from this surface's latest observation. - * @param value - Replacement value as JSON; supported values are defined by the platform provider. - * @returns dispatch evidence. - */ - setValue(elementId: CuaElementId, value: JsonValue): Promise - /** - * Select one literal text occurrence through writable AXSelectedTextRange and verify its range. - * @param elementId - Text control element id from this surface's latest observation. - * @param text - Nonempty literal text; multiple matches require adjacent context. - * @param options - Adjacent prefix/suffix and selection or cursor placement. - * @returns Dispatch evidence; rejects unsupported, ambiguous or changed text without keyboard fallback. - */ - selectText(elementId: CuaElementId, text: string, options?: CuaSelectTextOptions): Promise /** * Scroll at a screenshot-local point or integer ratio. * @param options - point, coordinate mode and line deltas; at least one delta must be nonzero. @@ -186,14 +185,7 @@ export interface CuaSurface { */ drag(from: readonly [number, number], to: readonly [number, number], options?: CuaDragOptions): Promise } -/** One explicit window; keyboard operations remain app-scoped because they follow focus. */ -export interface CuaWindow extends CuaSurface { readonly windowId: number } -/** - * Native app handle supporting direct observations and actions without window enumeration. - * Screenshots and coordinates reuse the latest AX observation's focused on-screen, - * nonzero-size window, or require exactly one such window. Read fresh AX to follow - * focus changes; explicit window objects retain their selected target. - */ +/** Application-level accessibility, window discovery, keyboard input and clipboard operations. */ export interface CuaApp extends CuaSurface { readonly pid: number readonly appHandle: CuaAppHandle @@ -211,7 +203,7 @@ export interface CuaApp extends CuaSurface { /** * Observe a specific window. * @param windowId - ID obtained from getWindows. - * @returns a reusable window object with its initial AX state retained silently; getAXState emits it. + * @returns A window object with its initial accessibility tree read silently; this updates the application's current snapshot. */ getWindow(windowId: number): Promise /** @@ -265,7 +257,7 @@ export interface CuaApi { /** * Select an app, obtain consent, launch it in the background if necessary, and silently retain initial AX state. * @param selector - Exact name, bundle ID, absolute application path or running PID string; ambiguous names reject. - * @returns a handle tied to this process lifetime. + * @returns An application handle tied to this process lifetime; its initial tree read becomes the current snapshot. */ getApp(selector: string): Promise } diff --git a/packages/experimental/cua/src/window-api.ts b/packages/experimental/cua/src/window-api.ts index 2eadb16b33..60c3ad1035 100644 --- a/packages/experimental/cua/src/window-api.ts +++ b/packages/experimental/cua/src/window-api.ts @@ -1,6 +1,6 @@ /** Window facade using the same observed surface operations as the application facade. */ import { z } from 'zod' -import { createCuaSurface, keyChord } from './api.ts' +import { createCuaObjects, keyChord } from './api.ts' import { ActionResult, CapabilitiesResult } from './protocol.ts' import { LaunchEntriesResult, LaunchResult, SelectedWindowsResult, WindowDiscoveryResult, WindowRequest } from './window-protocol.ts' import type { CuaAppHandle, CuaObservationOptions, CuaObservationSink } from './types.ts' @@ -43,23 +43,23 @@ export function createCuaWindowApi(dispatch: CuaWindowDispatch, sink?: CuaObserv const { windowId, pid } = selectedWindow const windowHandle = selectedWindow.windowHandle as CuaWindowHandle const appHandle = selectedWindow.windowHandle as CuaAppHandle - const surface = createCuaSurface(async (operation) => { - const params = z.record(z.string(), z.json()).parse(operation.params) - delete params.appHandle - delete params.windowId - const result = await call(operation.method, { ...params, windowHandle }) - if (operation.method !== 'getState') return result - const state = z.record(z.string(), z.json()).parse(result) - return { ...state, appHandle } - }, sink, appHandle, pid, windowId) const window: CuaDesktopWindow = { - ...surface, windowId, windowHandle, + ...createCuaObjects(async (operation) => { + const params = z.record(z.string(), z.json()).parse(operation.params) + delete params.appHandle + delete params.windowId + const result = await call(operation.method, { ...params, windowHandle }) + if (operation.method !== 'getState') return result + const state = z.record(z.string(), z.json()).parse(result) + return { ...state, appHandle } + }, sink).createWindow(appHandle, pid, windowId), + windowHandle, async activate() { return ActionResult.parse(await call('activate', { windowHandle })) }, async typeText(text) { return ActionResult.parse(await call('typeText', { windowHandle, text })) }, async paste(text, options) { return ActionResult.parse(await call('paste', { windowHandle, text, ...options })) }, async pressKey(key) { return ActionResult.parse(await call('pressKey', { windowHandle, ...keyChord(key) })) }, } - await window.getAXState({ emit: false }) + await window.getAccessibilityTree({ emit: false }) windows.push(window) } return windows diff --git a/packages/experimental/cua/src/window-types.ts b/packages/experimental/cua/src/window-types.ts index 133414c5b7..c92d066166 100644 --- a/packages/experimental/cua/src/window-types.ts +++ b/packages/experimental/cua/src/window-types.ts @@ -2,7 +2,7 @@ import type { Branded } from '@deepseek-ai/dsh-brand' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import type { z } from 'zod' -import type { CuaActionResult, CuaCapabilities, CuaObservationOptions, CuaPasteOptions, CuaSurface, CuaWindowInfo } from './types.ts' +import type { CuaActionResult, CuaCapabilities, CuaObservationOptions, CuaPasteOptions, CuaWindow, CuaWindowInfo } from './types.ts' import type { LaunchEntriesResult, WindowRequest } from './window-protocol.ts' /** Connection-owned grant for exactly one observed window lifetime. */ @@ -14,7 +14,7 @@ export type CuaWindowDispatch = (operation: CuaWindowOperation) => Promise[number] /** Selected window; keyboard and clipboard operations require this window in front. */ -export interface CuaDesktopWindow extends CuaSurface { +export interface CuaDesktopWindow extends CuaWindow { windowId: number windowHandle: CuaWindowHandle /** Bring this window forward. @returns Dispatch evidence, not application success. */ @@ -32,7 +32,7 @@ export interface CuaWindowApi { getCapabilities(options?: CuaObservationOptions): Promise /** Discover top-level windows without reading AX. @param options Output options. @returns Window metadata. */ listWindows(options?: CuaObservationOptions): Promise - /** Authorize selected windows and read their initial AX silently. A failure may leave earlier grants in place. + /** Authorize selected windows and read their initial accessibility trees silently. A failure may leave earlier grants in place. * @param ids IDs from listWindows; duplicates reject. @returns Windows in requested order. */ getWindows(ids: readonly number[]): Promise diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index 852e19fa9c..f1bf35db08 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest' import { brandString } from '@deepseek-ai/dsh-brand' import type { JsonValue } from '@deepseek-ai/dsh-util-values' import { createCuaApi } from '../src/api.ts' -import type { CuaApp, CuaElementId, CuaOperation, CuaScreenshot } from '../src/types.ts' +import type { CuaApp, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts' /** Element ids are opaque strings; these fixtures use the macOS decimal form. */ const element = (id: string): CuaElementId => brandString(id) @@ -13,7 +13,7 @@ const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG const bounds = { x: -640, y: 100, width: 640, height: 480 } const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor' } const delivered = { dispatched: true } -const capabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false } +const capabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false } function windowInfo(windowId: number) { return { windowId, pid: 17, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null } @@ -77,9 +77,10 @@ describe('application discovery', () => { const app = await f.api.getApp('Editor') expect(app.pid).toBe(17) expect(app.appHandle).toBe('app-17') - expect(await app.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 }) + const window = await app.getWindow(23) + expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 }) expect(f.dispatch.mock.calls.map(([operation]) => operation.method)).toEqual([ - 'getCapabilities', 'listApps', 'getApp', 'getState', 'screenshot', + 'getCapabilities', 'listApps', 'getApp', 'getState', 'getState', 'screenshot', ]) expect(await app.getWindows()).toEqual([windowInfo(23)]) expect(f.sink.text).not.toHaveBeenCalled() @@ -154,62 +155,63 @@ describe('hover across observations', () => { it('moves to an AX element then into a popup without inserting clicks or restoring a point', async () => { const f = fixture() const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) f.dispatch.mockClear() - expect(await app.hover(element('0'))).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false }) - await app.getAXStateAndScreenshot({ emit: false }) - expect(await app.hover([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false }) + expect(await window.hover(element('0'))).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false }) + await window.getAccessibilityTree({ emit: false }) + await window.getScreenshot({ emit: false }) + expect(await window.hover([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false }) expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([ - { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } }, - { method: 'getState', params: { appHandle: 'app-17' } }, + { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0' } }, + { method: 'getState', params: { appHandle: 'app-17', windowId: 23 } }, { method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } }, - { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 30, y: 40 } }, + { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 23, x: 30, y: 40 } }, ]) - const window = await app.getWindow(24) - await window.hover([5, 6]) + const other = await app.getWindow(24) + await other.hover([5, 6]) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'hover', params: { - appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 24, x: 5, y: 6, + appHandle: 'app-17', snapshotId: 'snapshot-4', windowId: 24, x: 5, y: 6, } }) f.dispatch.mockRejectedValueOnce(new Error('WINDOW_CHANGED')) - await expect(window.hover([5, 6])).rejects.toThrow('WINDOW_CHANGED') + await expect(other.hover([5, 6])).rejects.toThrow('WINDOW_CHANGED') expect(f.dispatch).toHaveBeenCalledTimes(7) }) }) describe('pointer buttons and modifiers', () => { - it.each([0, 800])('preserves %d ms for app and window clicks at elements and coordinates', async (holdMs) => { + it.each([0, 800])('preserves %d ms for window clicks at elements and coordinates', async (holdMs) => { const f = fixture() const app = await f.api.getApp('Editor') const window = await app.getWindow(23) - for (const surface of [app, window]) { - for (const target of [element('0'), [10, 20] as const]) { - await surface.click(target, { holdMs, mouseButton: 'right' }) - expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ method: 'click', params: { holdMs, mouseButton: 'right' } }) - f.dispatch.mockClear() - await expect(surface.click(target, { holdMs: -1 })).rejects.toThrow('holdMs') - expect(f.dispatch).not.toHaveBeenCalled() - } + for (const target of [element('0'), [10, 20] as const]) { + await window.click(target, { holdMs, mouseButton: 'right' }) + expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ method: 'click', params: { holdMs, mouseButton: 'right' } }) + f.dispatch.mockClear() + await expect(window.click(target, { holdMs: -1 })).rejects.toThrow('holdMs') + expect(f.dispatch).not.toHaveBeenCalled() } }) it('retains middle-button and modifier intent through click and drag dispatch', async () => { const f = fixture() const app = await f.api.getApp('Editor') - await app.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] }) + const window = await app.getWindow(23) + await window.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] }) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { - appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', mouseButton: 'middle', modifiers: ['shift'], + appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', mouseButton: 'middle', modifiers: ['shift'], } }) - await app.click([10, 20], { modifiers: ['command'] }) + await window.click([10, 20], { modifiers: ['command'] }) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { - appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, + appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1, modifiers: ['command'], } }) - await app.drag([10, 20], [30, 40], { mouseButton: 'middle', modifiers: ['shift', 'option'] }) + await window.drag([10, 20], [30, 40], { mouseButton: 'middle', modifiers: ['shift', 'option'] }) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'drag', params: { - appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, fromX: 10, fromY: 20, toX: 30, toY: 40, + appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', fromX: 10, fromY: 20, toX: 30, toY: 40, mouseButton: 'middle', modifiers: ['shift', 'option'], } }) const count = f.dispatch.mock.calls.length - await expect(app.click(element('0'), { mouseButton: 'middle', clickCount: 2 })).rejects.toThrow('Only left double-click') + await expect(window.click(element('0'), { mouseButton: 'middle', clickCount: 2 })).rejects.toThrow('Only left double-click') expect(f.dispatch).toHaveBeenCalledTimes(count) }) }) @@ -307,30 +309,87 @@ describe('element default invocation', () => { }) describe('application and window observations', () => { - it('keeps the latest element id scope separate for an app and each selected window', async () => { + it('shares the latest tree read across an application and each selected window', async () => { const f = fixture() f.setWindows([23, 24]) const app = await f.api.getApp('Editor') expect(await app.getWindows({ emit: false })).toEqual([windowInfo(23), windowInfo(24)]) const first = await app.getWindow(23) const second = await app.getWindow(24) - await app.click(element('0')) - await first.setValue(element('1'), 'First document') - await second.click(element('2')) + await app.setValue(element('1'), 'First document') + await first.click(element('2')) + await second.click(element('3')) expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([ - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } }, - { method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '1', value: 'First document' } }, + { method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '1', value: 'First document' } }, { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2' } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3' } }, ]) expect(first.windowId).toBe(23) expect(second.windowId).toBe(24) expect(f.sink.text.mock.calls.map(([text]) => text)).toEqual([]) - await expect(second.getAXState({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4') + await expect(second.getAccessibilityTree({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4') await second.click(element('3')) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3' } }) + await app.getAccessibilityTree({ emit: false }) + await first.invoke(element('4')) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4' } }) expect(f.sink.text).not.toHaveBeenCalled() }) + it('keeps separate snapshots for different application handles', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockResolvedValueOnce({ appHandle: 'app-18', pid: 18, name: 'Utility' }) + f.dispatch.mockResolvedValueOnce(state({ appHandle: 'app-18', pid: 18, snapshotId: 'other', windows: [] })) + const other = await f.api.getApp('Utility') + await app.setValue(element('0'), 'Editor') + await other.setValue(element('0'), 'Utility') + expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([ + { method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', value: 'Editor' } }, + { method: 'setValue', params: { appHandle: 'app-18', snapshotId: 'other', elementId: '0', value: 'Utility' } }, + ]) + }) + + it('shares one current snapshot between repeated handles for the same application', async () => { + const f = fixture() + const first = await f.api.getApp('Editor') + const second = await f.api.getApp('Editor') + expect(second.appHandle).toBe(first.appHandle) + await first.setValue(element('1'), 'Shared') + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'setValue', params: { + appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '1', value: 'Shared', + } }) + }) + + it('exposes screenshots and pointer input only on explicit windows', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + expect(app).toHaveProperty('getAccessibilityTree') + for (const method of ['getScreenshot', 'click', 'hover', 'scroll', 'drag']) expect(app).not.toHaveProperty(method) + const window = await app.getWindow(23) + for (const method of ['getScreenshot', 'click', 'hover', 'scroll', 'drag']) expect(window).toHaveProperty(method) + }) + + it('attaches the shared snapshot to coordinates only within its window scope', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + const first = await app.getWindow(23) + await first.click([10, 20]) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { + appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1, + } }) + const second = await app.getWindow(24) + await first.click([10, 20]) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { + appHandle: 'app-17', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1, + } }) + await second.click([10, 20]) + expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-3', windowId: 24 } }) + await app.getAccessibilityTree({ emit: false }) + await first.click([10, 20]) + expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-4', windowId: 23 } }) + }) + it('keeps a validated AXless window usable for screenshots and coordinate input', async () => { const f = fixture() const app = await f.api.getApp('Editor') @@ -357,17 +416,7 @@ describe('application and window observations', () => { expect(f.dispatch.mock.calls.slice(calls).map(([operation]) => operation.method)).toEqual(['getState']) }) - it.each([{ ids: [] }, { ids: [23, 24] }])('requires an explicit window when discovery yields $ids', async ({ ids }) => { - const f = fixture() - f.setWindows(ids) - const app = await f.api.getApp('Editor') - const calls = f.dispatch.mock.calls.length - await expect(app.click([1, 2])).rejects.toThrow('Select a window') - await expect(app.getScreenshot()).rejects.toThrow('Select a window') - expect(f.dispatch).toHaveBeenCalledTimes(calls) - }) - - it('selects the one visible nonzero window while preserving menu and hidden windows in discovery', async () => { + it('captures and clicks the explicitly selected window among several', async () => { const f = fixture() const windows = [ windowInfo(23), @@ -378,46 +427,28 @@ describe('application and window observations', () => { f.setWindowRecords(windows) const app = await f.api.getApp('Editor') expect(await app.getWindows({ emit: false })).toEqual(windows) - await app.getScreenshot() + const window = await app.getWindow(23) + await window.getScreenshot() expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } }) - await app.click([10, 20]) + await window.click([10, 20]) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { - appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1, + appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1, } }) const hidden = await app.getWindow(25) await hidden.click([10, 20]) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { - appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 25, x: 10, y: 20, mouseButton: 'left', clickCount: 1, + appHandle: 'app-17', windowId: 25, snapshotId: 'snapshot-3', x: 10, y: 20, mouseButton: 'left', clickCount: 1, } }) }) - it('requires explicit selection when every window is hidden or has zero size', async () => { + it('captures a window without reading its accessibility tree', async () => { const f = fixture() - f.setWindowRecords([ - { ...windowInfo(24), bounds: { ...bounds, width: 0, height: 0 } }, - { ...windowInfo(25), isOnScreen: false }, - ]) const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) const calls = f.dispatch.mock.calls.length - await expect(app.getScreenshot()).rejects.toThrow('Select a window') - expect(f.dispatch).toHaveBeenCalledTimes(calls) - }) - - it('captures the focused application window when several visible windows exist', async () => { - const f = fixture() - const app = await f.api.getApp('Editor') - f.dispatch.mockResolvedValueOnce(state({ focusedWindowId: 24, windows: [windowInfo(23), windowInfo(24)] })) - await app.getAXState() - await app.getScreenshot() - expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 24 } }) - }) - - it('does not select a focused window absent from the application observation', async () => { - const f = fixture() - const app = await f.api.getApp('Editor') - f.dispatch.mockResolvedValueOnce(state({ focusedWindowId: 99, windows: [windowInfo(23), windowInfo(24)] })) - await app.getAXState() - await expect(app.getScreenshot()).rejects.toThrow('Select a window') + await window.getScreenshot() + expect(f.dispatch).toHaveBeenCalledTimes(calls + 1) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } }) }) const identityMismatches: Record[] = [ @@ -427,9 +458,9 @@ describe('application and window observations', () => { const f = fixture() const app = await f.api.getApp('Editor') f.dispatch.mockResolvedValueOnce(state(identity)) - await expect(app.getAXState()).rejects.toThrow('another application or window') + await expect(app.getAccessibilityTree()).rejects.toThrow('another application or window') const calls = f.dispatch.mock.calls.length - await expect(app.click(element('0'))).rejects.toThrow('Read getAXState') + await expect(app.setValue(element('0'), 'stale')).rejects.toThrow('Read getAccessibilityTree') expect(f.dispatch).toHaveBeenCalledTimes(calls) }) @@ -438,22 +469,40 @@ describe('application and window observations', () => { const app = await f.api.getApp('Editor') const otherWindow = { ...windowInfo(23), pid: 18 } f.dispatch.mockResolvedValueOnce(state({ windows: [otherWindow] })) - await expect(app.getAXState()).rejects.toThrow('windows belong to another application') + await expect(app.getAccessibilityTree()).rejects.toThrow('windows belong to another application') f.dispatch.mockResolvedValueOnce([otherWindow]) await expect(app.getWindows()).rejects.toThrow('windows belong to another application') expect(f.sink.text).not.toHaveBeenCalled() }) - it('clears a failed refresh and obtains new state before a later screenshot', async () => { + it('accepts coordinate input without a current snapshot', async () => { const f = fixture() const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) + f.dispatch.mockRejectedValueOnce(new Error('Native app is busy')) + await expect(window.getAccessibilityTree()).rejects.toThrow('Native app is busy') + await window.click([10, 20]) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { + appHandle: 'app-17', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1, + } }) + }) + + it('clears a failed refresh and requires a new read before element operations', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) const failure = new Error('Native app is busy') f.dispatch.mockRejectedValueOnce(failure) - await expect(app.getAXState()).rejects.toBe(failure) - await expect(app.setValue(element('0'), 'stale')).rejects.toThrow('Read getAXState') + await expect(app.getAccessibilityTree()).rejects.toBe(failure) + await expect(window.setValue(element('0'), 'stale')).rejects.toThrow('Read getAccessibilityTree') + await window.getAccessibilityTree({ emit: false }) + f.dispatch.mockRejectedValueOnce(failure) + await expect(window.getAccessibilityTree()).rejects.toBe(failure) + await expect(app.invoke(element('0'))).rejects.toThrow('Read getAccessibilityTree') f.sink.text.mockClear() - await app.getScreenshot() - expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation.method)).toEqual(['getState', 'screenshot']) + const calls = f.dispatch.mock.calls.length + await window.getScreenshot() + expect(f.dispatch.mock.calls.slice(calls).map(([operation]) => operation.method)).toEqual(['screenshot']) expect(f.sink.text).not.toHaveBeenCalled() expect(f.sink.image).toHaveBeenCalledTimes(1) }) @@ -466,13 +515,15 @@ describe('screenshots', () => { const app = await f.api.getApp('Editor') const window = await app.getWindow(24) f.sink.text.mockClear() - const observation = await window.getAXStateAndScreenshot() - expect(observation.state).toBe('Tree 3') - expect(observation.screenshot).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 24 }) + const tree = await window.getAccessibilityTree() + const image = await window.getScreenshot() + expect(tree).toBe('Tree 3') + expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 24 }) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 24 } }) expect(f.sink.text).toHaveBeenCalledWith('Tree 3') - expect(f.sink.image).toHaveBeenCalledWith(observation.screenshot) - await window.getAXStateAndScreenshot({ emit: false }) + expect(f.sink.image).toHaveBeenCalledWith(image) + await window.getAccessibilityTree({ emit: false }) + await window.getScreenshot({ emit: false }) expect(f.sink.text).toHaveBeenCalledTimes(1) expect(f.sink.image).toHaveBeenCalledTimes(1) }) @@ -480,17 +531,19 @@ describe('screenshots', () => { it('rejects a screenshot captured from another window before emitting it', async () => { const f = fixture() const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) f.dispatch.mockResolvedValueOnce(screenshot({ windowId: 24 })) - await expect(app.getScreenshot()).rejects.toThrow('another window') + await expect(window.getScreenshot()).rejects.toThrow('another window') expect(f.sink.image).not.toHaveBeenCalled() }) it.each(['image/jpeg', 'image/webp', 'image/gif'])('preserves Host-admitted %s bytes and metadata in the return value and image sink', async (mimeType) => { const f = fixture() const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) const data = Buffer.from('admitted request image').toString('base64') f.dispatch.mockResolvedValueOnce(screenshot({ mimeType, data, pixelWidth: 320, pixelHeight: 240 })) - const image = await app.getScreenshot() + const image = await window.getScreenshot() expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(data, 'base64')), mimeType, pixelWidth: 320, pixelHeight: 240, windowId: 23 }) expect(f.sink.image).toHaveBeenCalledExactlyOnceWith(image) }) @@ -501,21 +554,22 @@ describe('input actions', () => { it('passes explicit relative mode to every coordinate gesture and rejects it on element ids', async () => { const f = fixture() const app = await f.api.getApp('Editor') - await app.click([500, 250], { relative: true }) - await app.hover([500, 250], { relative: true }) - await app.drag([0, 0], [999, 999], { relative: true }) - await app.scroll({ x: 500, y: 250, relative: true, deltaLinesX: 0, deltaLinesY: -1 }) + const window = await app.getWindow(23) + await window.click([500, 250], { relative: true }) + await window.hover([500, 250], { relative: true }) + await window.drag([0, 0], [999, 999], { relative: true }) + await window.scroll({ x: 500, y: 250, relative: true, deltaLinesX: 0, deltaLinesY: -1 }) expect(f.dispatch.mock.calls.slice(-4).map(([request]) => request.params)).toEqual([ expect.objectContaining({ x: 500, y: 250, relative: true }), expect.objectContaining({ x: 500, y: 250, relative: true }), expect.objectContaining({ fromX: 0, toX: 999, relative: true }), expect.objectContaining({ x: 500, y: 250, relative: true }), ]) - await expect(app.click(element('1'), { relative: true })).rejects.toThrow() - await expect(app.hover(element('1'), { relative: true })).rejects.toThrow() + await expect(window.click(element('1'), { relative: true })).rejects.toThrow() + await expect(window.hover(element('1'), { relative: true })).rejects.toThrow() }) - it('keeps the selected app and window when scroll options include another target identity', async () => { + it('keeps the selected window when scroll options include another target identity', async () => { const f = fixture() const app = await f.api.getApp('Editor') const window = await app.getWindow(23) @@ -534,57 +588,61 @@ describe('input actions', () => { it('dispatches window-point gestures and reports delivery without claiming the action succeeded', async () => { const f = fixture() const app = await f.api.getApp('Editor') - expect(await app.click([10, 20], { mouseButton: 'right' })).toEqual(delivered) - expect(await app.click([11, 21], { clickCount: 2 })).toEqual(delivered) - expect(await app.scroll({ x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 })).toEqual(delivered) - expect(await app.drag([1, 2], [30, 40])).toEqual(delivered) + const window = await app.getWindow(23) + expect(await window.click([10, 20], { mouseButton: 'right' })).toEqual(delivered) + expect(await window.click([11, 21], { clickCount: 2 })).toEqual(delivered) + expect(await window.scroll({ x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 })).toEqual(delivered) + expect(await window.drag([1, 2], [30, 40])).toEqual(delivered) expect(f.dispatch.mock.calls.slice(-4).map(([operation]) => operation)).toEqual([ - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'right', clickCount: 1 } }, - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 11, y: 21, mouseButton: 'left', clickCount: 2 } }, - { method: 'scroll', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } }, - { method: 'drag', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, fromX: 1, fromY: 2, toX: 30, toY: 40 } }, + { method: 'click', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'right', clickCount: 1 } }, + { method: 'click', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 11, y: 21, mouseButton: 'left', clickCount: 2 } }, + { method: 'scroll', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } }, + { method: 'drag', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', fromX: 1, fromY: 2, toX: 30, toY: 40 } }, ]) }) - const invalidActions: Array<[string, (app: CuaApp) => Promise]> = [ - ['right double click', app => app.click([1, 2], { mouseButton: 'right', clickCount: 2 })], - ['right AX double click', app => app.click(element('0'), { mouseButton: 'right', clickCount: 2 })], - ['empty hover element id', app => app.hover(element(''))], - ['negative hover point', app => app.hover([-1, 0])], - ['nonfinite hover point', app => app.hover([NaN, 0])], - ['empty click element id', app => app.click(element(''))], - ['nonfinite coordinate', app => app.click([NaN, 0])], - ['negative coordinate', app => app.click([-1, 0])], - ['empty value element id', app => app.setValue(element(''), 'value')], - ['zero scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0, deltaLinesY: 0 })], - ['oversized scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })], - ['fractional scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })], - ['negative drag point', app => app.drag([0, 0], [-1, 3])], - ['empty chord', app => app.pressKey('')], - ['missing key', app => app.pressKey('super+')], - ['empty plus modifier', app => app.pressKey('++')], - ['extra plus delimiter', app => app.pressKey('cmd+++')], - ['unknown modifier', app => app.pressKey('unknown+a')], + type Surfaces = { app: CuaApp; window: CuaWindow } + const invalidActions: Array<[string, (surfaces: Surfaces) => Promise]> = [ + ['right double click', ({ window }) => window.click([1, 2], { mouseButton: 'right', clickCount: 2 })], + ['right AX double click', ({ window }) => window.click(element('0'), { mouseButton: 'right', clickCount: 2 })], + ['empty hover element id', ({ window }) => window.hover(element(''))], + ['negative hover point', ({ window }) => window.hover([-1, 0])], + ['nonfinite hover point', ({ window }) => window.hover([NaN, 0])], + ['empty click element id', ({ window }) => window.click(element(''))], + ['nonfinite coordinate', ({ window }) => window.click([NaN, 0])], + ['negative coordinate', ({ window }) => window.click([-1, 0])], + ['empty value element id', ({ window }) => window.setValue(element(''), 'value')], + ['zero scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0, deltaLinesY: 0 })], + ['oversized scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })], + ['fractional scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })], + ['negative drag point', ({ window }) => window.drag([0, 0], [-1, 3])], + ['empty chord', ({ app }) => app.pressKey('')], + ['missing key', ({ app }) => app.pressKey('super+')], + ['empty plus modifier', ({ app }) => app.pressKey('++')], + ['extra plus delimiter', ({ app }) => app.pressKey('cmd+++')], + ['unknown modifier', ({ app }) => app.pressKey('unknown+a')], ] - it.each<[string, (app: CuaApp) => Promise]>([ - ['getState windowId: Too small: expected number to be >0', app => app.getWindow(0)], - ['selectText text: Too small: expected string to have >=1 characters', app => app.selectText(element('0'), '')], - ['hover x: Invalid input: expected number, received NaN', app => app.hover([NaN, 0])], - ['hover x: Too big: expected number to be <=999', app => app.hover([1000, 0], { relative: true })], - ['click relative: not accepted for these arguments', app => app.click(element('650'), { relative: true })], + it.each<[string, (surfaces: Surfaces) => Promise]>([ + ['getState windowId: Too small: expected number to be >0', ({ app }) => app.getWindow(0)], + ['selectText text: Too small: expected string to have >=1 characters', ({ app }) => app.selectText(element('0'), '')], + ['hover x: Invalid input: expected number, received NaN', ({ window }) => window.hover([NaN, 0])], + ['hover x: Too big: expected number to be <=999', ({ window }) => window.hover([1000, 0], { relative: true })], + ['click relative: not accepted for these arguments', ({ window }) => window.click(element('650'), { relative: true })], ])('reports a readable validation failure: %s', async (message, run) => { const f = fixture() const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) f.dispatch.mockClear() - await expect(run(app)).rejects.toMatchObject({ name: 'Error', message }) + await expect(run({ app, window })).rejects.toMatchObject({ name: 'Error', message }) expect(f.dispatch).not.toHaveBeenCalled() }) it.each(invalidActions)('rejects %s without invoking native input', async (_label, run) => { const f = fixture() const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) const calls = f.dispatch.mock.calls.length - await expect(run(app)).rejects.toThrow() + await expect(run({ app, window })).rejects.toThrow() expect(f.dispatch).toHaveBeenCalledTimes(calls) }) @@ -621,22 +679,24 @@ describe('input actions', () => { it('retains the snapshot identity for element right and double clicks', async () => { const f = fixture() const app = await f.api.getApp('Editor') - await app.click(element('12'), { mouseButton: 'right' }) - await app.click(element('13'), { clickCount: 2 }) + const window = await app.getWindow(23) + await window.click(element('12'), { mouseButton: 'right' }) + await window.click(element('13'), { clickCount: 2 }) expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([ - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '12', mouseButton: 'right' } }, - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '13', clickCount: 2 } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', mouseButton: 'right' } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', clickCount: 2 } }, ]) }) it('dispatches an opaque provider element id unchanged', async () => { const f = fixture() const app = await f.api.getApp('Editor') - await app.click(element('42.12345')) - await app.hover(element('42.12345')) + const window = await app.getWindow(23) + await window.click(element('42.12345')) + await window.hover(element('42.12345')) expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([ - { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345' } }, - { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345' } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } }, + { method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } }, ]) }) @@ -651,13 +711,14 @@ describe('input actions', () => { it('propagates native failures once and rejects malformed delivery claims', async () => { const f = fixture() const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) const failure = new Error('Application was restarted; select it again') f.dispatch.mockRejectedValueOnce(failure) await expect(app.typeText('hello')).rejects.toBe(failure) - expect(f.dispatch).toHaveBeenCalledTimes(3) - f.dispatch.mockResolvedValueOnce({ dispatched: 'yes' }) - await expect(app.click(element('0'))).rejects.toThrow() expect(f.dispatch).toHaveBeenCalledTimes(4) + f.dispatch.mockResolvedValueOnce({ dispatched: 'yes' }) + await expect(window.click(element('0'))).rejects.toThrow() + expect(f.dispatch).toHaveBeenCalledTimes(5) }) it('rejects invalid selection inputs before dispatch', async () => { @@ -674,8 +735,8 @@ describe('input actions', () => { it('expands menu contents and details only for the requested observation', async () => { const f = fixture() const app = await f.api.getApp('Editor') - await app.getAXState({ includeMenus: true, details: true, emit: false }) + await app.getAccessibilityTree({ includeMenus: true, details: true, emit: false }) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'getState', params: { appHandle: selected.appHandle, includeMenus: true, details: true } }) - await app.getAXState({ includeMenus: false, details: false }) + await app.getAccessibilityTree({ includeMenus: false, details: false }) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'getState', params: { appHandle: selected.appHandle, includeMenus: false, details: false } }) }) diff --git a/packages/experimental/cua/tests/bridge.spec.ts b/packages/experimental/cua/tests/bridge.spec.ts index 429444fcba..ed6e624561 100644 --- a/packages/experimental/cua/tests/bridge.spec.ts +++ b/packages/experimental/cua/tests/bridge.spec.ts @@ -19,20 +19,25 @@ const snapshot = { } it('uses the current tool binding when a retained app is called by another execution', async () => { - const first = vi.fn(async (operation: CuaOperation): Promise => operation.method === 'getApp' ? selected : snapshot) + const first = vi.fn(async (operation: CuaOperation): Promise => { + if (operation.method === 'getApp') return selected + if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null } + return snapshot + }) let current: CuaToolBinding = first const text = vi.fn() const cua = connectCua(() => current, text) const app = await cua.getApp('Editor') + const window = await app.getWindow(23) const second = vi.fn(async (): Promise => ({ dispatched: true })) current = second - await expect(app.click(element('0'))).resolves.toEqual({ dispatched: true }) - expect(first).toHaveBeenCalledTimes(2) + await expect(window.click(element('0'))).resolves.toEqual({ dispatched: true }) + expect(first).toHaveBeenCalledTimes(3) expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } }) expect(text).not.toHaveBeenCalled() current = first - await app.getAXState() + await app.getAccessibilityTree() expect(text).toHaveBeenCalledWith('0 button') }) @@ -57,16 +62,17 @@ it.each([null, 'text only', { content: [{ type: 'text', text: '{}' }] }])('rejec it('returns screenshot bytes and metadata with one image-bearing tool call and no image text output', async () => { const tool = vi.fn(async (operation: CuaOperation): Promise => { if (operation.method === 'getApp') return selected - if (operation.method === 'getState') return snapshot + if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null } return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 23, bounds, coordinateSpace: 'screenshot-pixels', } }) const text = vi.fn() const app = await connectCua(() => tool, text).getApp('Editor') + const window = await app.getWindow(23) text.mockClear() - expect(await app.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 }) - expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getApp', 'getState', 'screenshot']) + expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 }) + expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getApp', 'getState', 'getState', 'screenshot']) expect(text).not.toHaveBeenCalled() }) diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index 5eff8eded0..4ca924336d 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -1,7 +1,7 @@ /** Model-authored operation JSON cannot supply host configuration or bypass retained identities. */ import { expect, it } from 'vitest' import type { JsonValue } from '@deepseek-ai/dsh-util-values' -import { Request, ScreenshotResult } from '../src/protocol.ts' +import { CapabilitiesResult, Request, ScreenshotResult } from '../src/protocol.ts' const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [ { reason: 'raw process ID instead of a selected app handle', request: { method: 'typeText', params: { pid: 17, text: 'hello' } } }, @@ -155,3 +155,13 @@ it.each(['click', 'hover', 'scroll', 'drag'])('validates 0–999 integer coordin expect(Request.safeParse({ method, params: { ...params, relative: false, [field]: 1000 } }).success).toBe(true) } }) + +it.each([true, false])('requires the reported application-level accessibility capability: %s', (appLevelAccessibility) => { + const capabilities = { + platform: 'macos', permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility, + sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false, + } + expect(CapabilitiesResult.parse(capabilities)).toEqual(capabilities) + const { appLevelAccessibility: _omitted, ...missing } = capabilities + expect(CapabilitiesResult.safeParse(missing).success).toBe(false) +}) diff --git a/packages/experimental/cua/tests/window-api.spec.ts b/packages/experimental/cua/tests/window-api.spec.ts index 7a3766aed3..44197b4029 100644 --- a/packages/experimental/cua/tests/window-api.spec.ts +++ b/packages/experimental/cua/tests/window-api.spec.ts @@ -11,6 +11,7 @@ import type { CuaWindowOperation } from '../src/window-types.ts' const element = (id: string): CuaElementId => brandString(id) const metadata = { windowId: 7, pid: 10, appName: 'Editor', title: 'Document', bounds: { x: -100, y: 0, width: 800, height: 600 }, isOnScreen: true, onCurrentSpace: null } +const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC' function fixture() { const text = vi.fn() const dispatch = vi.fn(async (request: CuaWindowOperation): Promise => { @@ -20,6 +21,8 @@ function fixture() { case 'getState': return { windowHandle: 'window-7', windowId: 7, pid: 10, snapshotId: 'state-1', tree: 'AX window', windows: [metadata], truncated: false } case 'findApps': return [{ id: 'shell-entry', displayName: 'Settings', kind: 'shell' }] case 'listActions': return ['AXPress'] + case 'hover': return { dispatched: true, hoverRoute: 'accessibility', systemPointerMoved: false } + case 'screenshot': return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 7, bounds: metadata.bounds, coordinateSpace: 'screenshot-pixels' } default: return { dispatched: true } } }) @@ -39,9 +42,15 @@ it('binds all observation and keyboard operations to a selected window and suppr expect(window).toBeDefined() if (!window) throw new Error('Missing selected window') expect(f.text).not.toHaveBeenCalled() - await window.getAXState() + expect(f.dispatch.mock.calls[1]?.[0]).toEqual({ method: 'getState', params: { windowHandle: 'window-7' } }) + await window.getAccessibilityTree() expect(f.text).toHaveBeenCalledWith('AX window') await window.click(element('0')) + await window.click([1, 2]) + await window.hover(element('42.12345')) + await window.scroll({ x: 1, y: 2, deltaLinesX: 0, deltaLinesY: 1 }) + await window.drag([1, 2], [3, 4]) + expect(await window.getScreenshot()).toMatchObject({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 7 }) await window.activate() await window.typeText('hello') await window.paste('world') @@ -116,7 +125,7 @@ it('supports discovery without a sink and suppresses requested emissions', async await f.api.listWindows({ emit: false }) expect(f.text).not.toHaveBeenCalled() await createCuaWindowApi(f.dispatch).listWindows() - const capabilities = { platform: 'windows', permissions: { accessibility: true, screenRecording: true }, sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false } + const capabilities = { platform: 'windows', permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: false, sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false } f.dispatch.mockResolvedValueOnce(capabilities) expect(await f.api.getCapabilities()).toEqual(capabilities) }) diff --git a/snapshots/session/cua-native/workspace/native-helper.mjs b/snapshots/session/cua-native/workspace/native-helper.mjs index 2e4ab6426b..7c1de69f26 100755 --- a/snapshots/session/cua-native/workspace/native-helper.mjs +++ b/snapshots/session/cua-native/workspace/native-helper.mjs @@ -131,7 +131,7 @@ function createApplicationState(socket, record, application, captureBroker, conn const state = JSON.parse(readFileSync(new URL('./permissions.json', import.meta.url), 'utf8')) return { platform: 'darwin', ...state, helper: { pid: process.pid }, parent: { pid: process.ppid } } } - if (method === 'getCapabilities') return { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: true }, methods, coordinateSpace: 'window-points', browser: false } + if (method === 'getCapabilities') return { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: true, methods, coordinateSpace: 'window-points', browser: false } if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId }] if (method === 'getApp') { if (params.selector === 'Denied Fixture' || params.selector === 'Revocable Fixture') {