refactor(cua): separate window operations from accessibility surfaces

This commit is contained in:
Huo Yaoyuan committed 2026-11-04 13:22:42 +08:00
1 parent 0c4eb10d94
commit bfd5e9fccc
15 files changed
+428 -346

No files matched your search

@@ -193,6 +193,7 @@ actor AXWorker {
if method == "getCapabilities" {
return .result(.object(["platform": .string("macos"), "permissions": .object([
"accessibility": .bool(AXIsProcessTrusted()), "screenRecording": .bool(CGPreflightScreenCaptureAccess())]),
"appLevelAccessibility": .bool(true),
"sessionLocked": locked.map(JSONValue.bool) ?? .null,
"methods": .array(NativeProtocol.modelMethods.map(JSONValue.string)), "coordinateSpace": .string("window-points"), "browser": .bool(false)]))
}
@@ -173,7 +173,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
})
const retainedApp = await objectApi(agent).getApp('test.fixture')
const retainedWindow = await retainedApp.getWindow(7)
expect(await retainedWindow.getAXState({ emit: false })).toContain('Count: 1')
expect(await retainedWindow.getAccessibilityTree({ emit: false })).toContain('Count: 1')
const peer = await context.agents.create({ sessionId: SessionId('native-loader-peer'), agentOptions: { provider: 'native-fixture', model: 'vision' } })
const peerApp = await objectApi(peer.agent).getApp('test.fixture')
const peerWindow = await peerApp.getWindow(7)
@@ -182,7 +182,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
expect(screenshot).toMatchObject({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 7 })
expect(Buffer.from(screenshot.bytes).toString('base64')).toBe(png)
await peerWindow.click(element('0'))
expect(await retainedWindow.getAXState({ emit: false })).toContain('Count: 2')
expect(await retainedWindow.getAccessibilityTree({ emit: false })).toContain('Count: 2')
const sharedEvents = await readEvents()
expect(sharedEvents.filter(event => event.event === 'start')).toHaveLength(1)
expect(sharedEvents.filter(event => event.event === 'connect')).toHaveLength(2)
@@ -191,7 +191,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
expect((await readEvents()).filter(event => event.event === 'exit')).toHaveLength(0)
expect((await readEvents()).filter(event => event.event === 'disconnect')).toHaveLength(1)
await peerWindow.click(element('0'))
expect(await peerWindow.getAXState({ emit: false })).toContain('Count: 3')
expect(await peerWindow.getAccessibilityTree({ emit: false })).toContain('Count: 3')
await peer.dispose()
expect((await readEvents()).filter(event => event.event === 'exit')).toHaveLength(1)
@@ -84,7 +84,7 @@ it.skipIf(process.platform === 'win32' || !existsSync(bridge) || !existsSync(boo
expect(invalid.error?.message).not.toContain('ZodError')
expect(invalid.error?.message).not.toContain('"origin"')
expect(calls.filter(call => call.phase === 'invalid')).toEqual([])
const second = await run('await win.getAXState({emit:false}); await win.click("0"); await win.getAXState({emit:false});', 'second')
const second = await run('await win.getAccessibilityTree({emit:false}); await win.click("0"); await win.getAccessibilityTree({emit:false});', 'second')
expect(second.error).toBeUndefined()
expect(second.value).toContain('Count: 2')
expect(calls.filter(call => call.phase === 'second').map(call => call.method)).toEqual(['getState', 'click', 'getState'])
@@ -164,7 +164,7 @@ it('correlates a native request with its logged tool invocation', async () => {
it('reports screenshot pixel coordinates while preserving native capabilities', async () => {
const f = await setup()
const capabilities = { platform: 'macos', coordinateSpace: 'window-points', permissions: { accessibility: true, screenCapture: false } }
const capabilities = { platform: 'macos', coordinateSpace: 'window-points', permissions: { accessibility: true, screenCapture: false }, appLevelAccessibility: true }
native.request.mockResolvedValueOnce(capabilities)
const result = await f.execute('getCapabilities')
expect(result.isError).toBe(false)
+120 -119
View File
@@ -68,7 +68,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
async function emit(value: object, options?: CuaObservationOptions): Promise<void> {
if (options?.emit !== false) await sink?.text(JSON.stringify(value))
}
const objects = createCuaObjects(dispatch, sink)
const api: CuaApi = {
async getCapabilities(options) {
@@ -96,7 +96,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
async getApp(selector) {
const { appHandle, pid } = AppResult.parse(await call('getApp', { selector }))
const app: CuaApp = {
...createCuaSurface(dispatch, sink, appHandle, pid), pid, appHandle,
...objects.createSurface(appHandle, pid), pid, appHandle,
async activate() { return action('activate', { appHandle }) },
async getWindows(options) {
const windows = WindowsResult.parse(await call('listWindows', { appHandle }))
@@ -105,8 +105,8 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
return windows
},
async getWindow(windowId): Promise<CuaWindow> {
const window = { ...createCuaSurface(dispatch, sink, appHandle, pid, windowId), windowId }
await window.getAXState({ emit: false })
const window = objects.createWindow(appHandle, pid, windowId)
await window.getAccessibilityTree({ emit: false })
return window
},
async typeText(text) { return action('typeText', { appHandle, text }) },
@@ -115,7 +115,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
}) },
async pressKey(key) { return action('pressKey', { appHandle, ...keyChord(key) }) },
}
await app.getAXState({ emit: false })
await app.getAccessibilityTree({ emit: false })
return app
},
}
@@ -123,127 +123,128 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
}
/**
* Bind one observed surface to its authorized dispatcher.
* Create application and window objects sharing their latest observations.
* @param dispatch Logged operation dispatcher.
* @param sink Optional observation output.
* @param appHandle Native authorization handle.
* @param pid Observed process lifetime owner.
* @param selectedWindow Explicit window selection, or the macOS application scope.
* @returns Observation and input methods retaining only the latest AX state.
* @returns Surface and window constructors with observation state owned by this factory.
*/
export function createCuaSurface(dispatch: CuaDispatch, sink: CuaObservationSink | undefined,
appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface {
export function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | undefined): {
createSurface(appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface
createWindow(appHandle: CuaAppHandle, pid: number, windowId: number): CuaWindow
} {
const call = validatedCall(dispatch)
const action = async (method: CuaOperation['method'], params: Record<string, JsonValue>) => ActionResult.parse(await call(method, params))
let observed: ReturnType<typeof StateResult.parse> | undefined
function snapshotParams(): Record<string, JsonValue> {
if (!observed) throw new Error('Read getAXState before using this surface')
return { appHandle, snapshotId: observed.snapshotId }
const observed = new Map<CuaAppHandle, ReturnType<typeof StateResult.parse>>()
function elementParams(appHandle: CuaAppHandle): Record<string, JsonValue> {
const current = observed.get(appHandle)
if (current === undefined) throw new Error('Read getAccessibilityTree before using this surface')
return { appHandle, snapshotId: current.snapshotId }
}
function windowId(): number {
if (selectedWindow !== undefined) return selectedWindow
const windows = observed?.windows.filter(window => window.isOnScreen && window.bounds.width > 0 && window.bounds.height > 0)
const focused = windows?.find(window => window.windowId === observed?.focusedWindowId)
if (focused) return focused.windowId
const first = windows?.[0]
if (!first || windows.length !== 1) throw new Error('Select a window with app.getWindows() and app.getWindow(windowId) before this operation')
return first.windowId
function coordinateParams(appHandle: CuaAppHandle, windowId: number): Record<string, JsonValue> {
const current = observed.get(appHandle)
return {
appHandle, windowId,
...current !== undefined && (current.windowId === null || current.windowId === windowId)
? { snapshotId: current.snapshotId } : {},
}
}
const result: CuaSurface = {
async show() {
return ShowResult.parse(await call('show', { appHandle,
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
}))
},
async getAXState(options) {
observed = undefined
const state = StateResult.parse(await call('getState', { appHandle,
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus },
...options?.details === undefined ? {} : { details: options.details },
}))
if (state.appHandle !== appHandle || state.pid !== pid || state.windowId !== (selectedWindow ?? null)) {
throw new Error('Native observation belongs to another application or window')
}
if (state.windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application')
observed = state
if (options?.emit !== false) await sink?.text(state.tree)
return state.tree
},
async getScreenshot(options) {
if (!observed) await result.getAXState({ emit: false })
const target = windowId()
const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId: target }))
if (image.windowId !== target) throw new Error('Native screenshot belongs to another window')
const screenshot: CuaScreenshot = {
bytes: Uint8Array.from(Buffer.from(image.data, 'base64')),
mimeType: image.mimeType,
pixelWidth: image.pixelWidth,
pixelHeight: image.pixelHeight,
windowId: image.windowId,
}
if (options?.emit !== false) await sink?.image(screenshot)
return screenshot
},
async getAXStateAndScreenshot(options) {
const state = await result.getAXState(options)
return { state, screenshot: await result.getScreenshot(options) }
},
async listActions(elementId) {
return ActionNamesResult.parse(await call('listActions', { ...snapshotParams(), elementId }))
},
async performAction(elementId, name, argument) {
return action('performAction', { ...snapshotParams(), elementId, action: name,
...argument === undefined ? {} : { argument },
})
},
async invoke(elementId) {
return ActionResult.parse(await call('invoke', { ...snapshotParams(), elementId }))
},
async hover(target, options) {
const relative = options?.relative === undefined ? {} : { relative: options.relative }
if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...snapshotParams(), elementId: target, ...relative }))
const [x, y] = target
return HoverResult.parse(await call('hover', { ...snapshotParams(), windowId: windowId(), x, y, ...relative }))
},
async click(target, options) {
const mouseButton = options?.mouseButton ?? 'left'
const clickCount = options?.clickCount ?? 1
if (mouseButton !== 'left' && clickCount !== 1) throw new Error('Only left double-click is supported')
const modifiers = options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] }
const hold = options?.holdMs === undefined ? {} : { holdMs: options.holdMs }
const relative = options?.relative === undefined ? {} : { relative: options.relative }
if (typeof target === 'string') {
return action('click', { ...snapshotParams(), ...modifiers, ...hold, ...relative, elementId: target,
...mouseButton === 'left' ? {} : { mouseButton }, ...clickCount === 1 ? {} : { clickCount },
function createSurface(appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface {
return {
async show() {
return ShowResult.parse(await call('show', { appHandle,
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
}))
},
async getAccessibilityTree(options) {
observed.delete(appHandle)
const state = StateResult.parse(await call('getState', { appHandle,
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus },
...options?.details === undefined ? {} : { details: options.details },
}))
if (state.appHandle !== appHandle || state.pid !== pid || state.windowId !== (selectedWindow ?? null)) {
throw new Error('Native observation belongs to another application or window')
}
if (state.windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application')
observed.set(appHandle, state)
if (options?.emit !== false) await sink?.text(state.tree)
return state.tree
},
async listActions(elementId) {
return ActionNamesResult.parse(await call('listActions', { ...elementParams(appHandle), elementId }))
},
async performAction(elementId, name, argument) {
return action('performAction', { ...elementParams(appHandle), elementId, action: name,
...argument === undefined ? {} : { argument },
})
}
const [x, y] = target
return action('click', { ...snapshotParams(), windowId: windowId(), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative })
},
async setValue(elementId, value) { return action('setValue', { ...snapshotParams(), elementId, value }) },
async selectText(elementId, text, options) {
return action('selectText', { ...snapshotParams(), elementId, text,
...options?.prefix === undefined ? {} : { prefix: options.prefix },
...options?.suffix === undefined ? {} : { suffix: options.suffix },
...options?.selectionType === undefined ? {} : { selectionType: options.selectionType },
})
},
async scroll(options) {
if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero')
return action('scroll', {
...snapshotParams(), windowId: windowId(),
x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY,
...options.relative === undefined ? {} : { relative: options.relative },
})
},
async drag(from, to, options) {
return action('drag', { ...snapshotParams(), windowId: windowId(), fromX: from[0], fromY: from[1], toX: to[0], toY: to[1],
...options?.relative === undefined ? {} : { relative: options.relative },
...options?.mouseButton === undefined ? {} : { mouseButton: options.mouseButton },
...options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] },
})
},
},
async invoke(elementId) {
return ActionResult.parse(await call('invoke', { ...elementParams(appHandle), elementId }))
},
async setValue(elementId, value) { return action('setValue', { ...elementParams(appHandle), elementId, value }) },
async selectText(elementId, text, options) {
return action('selectText', { ...elementParams(appHandle), elementId, text,
...options?.prefix === undefined ? {} : { prefix: options.prefix },
...options?.suffix === undefined ? {} : { suffix: options.suffix },
...options?.selectionType === undefined ? {} : { selectionType: options.selectionType },
})
},
}
}
return result
function createWindow(appHandle: CuaAppHandle, pid: number, windowId: number): CuaWindow {
return {
...createSurface(appHandle, pid, windowId), windowId,
async getScreenshot(options) {
const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId }))
if (image.windowId !== windowId) throw new Error('Native screenshot belongs to another window')
const screenshot: CuaScreenshot = {
bytes: Uint8Array.from(Buffer.from(image.data, 'base64')),
mimeType: image.mimeType,
pixelWidth: image.pixelWidth,
pixelHeight: image.pixelHeight,
windowId: image.windowId,
}
if (options?.emit !== false) await sink?.image(screenshot)
return screenshot
},
async hover(target, options) {
const relative = options?.relative === undefined ? {} : { relative: options.relative }
if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...elementParams(appHandle), elementId: target, ...relative }))
const [x, y] = target
return HoverResult.parse(await call('hover', { ...coordinateParams(appHandle, windowId), x, y, ...relative }))
},
async click(target, options) {
const mouseButton = options?.mouseButton ?? 'left'
const clickCount = options?.clickCount ?? 1
if (mouseButton !== 'left' && clickCount !== 1) throw new Error('Only left double-click is supported')
const modifiers = options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] }
const hold = options?.holdMs === undefined ? {} : { holdMs: options.holdMs }
const relative = options?.relative === undefined ? {} : { relative: options.relative }
if (typeof target === 'string') {
return action('click', { ...elementParams(appHandle), ...modifiers, ...hold, ...relative, elementId: target,
...mouseButton === 'left' ? {} : { mouseButton }, ...clickCount === 1 ? {} : { clickCount },
})
}
const [x, y] = target
return action('click', { ...coordinateParams(appHandle, windowId), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative })
},
async scroll(options) {
if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero')
return action('scroll', {
...coordinateParams(appHandle, windowId),
x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY,
...options.relative === undefined ? {} : { relative: options.relative },
})
},
async drag(from, to, options) {
return action('drag', { ...coordinateParams(appHandle, windowId),
fromX: from[0], fromY: from[1], toX: to[0], toY: to[1],
...options?.relative === undefined ? {} : { relative: options.relative },
...options?.mouseButton === undefined ? {} : { mouseButton: options.mouseButton },
...options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] },
})
},
}
}
return { createSurface, createWindow }
}
+1 -1
View File
@@ -12,7 +12,7 @@ export type CuaToolBinding = (operation: CuaOperation) => Promise<JsonValue>
* Connect the object API without owning an evaluator or its lifetime.
* @param getTool - Resolves the current execution's tools.cua binding on every operation; do not capture a completed execution's binding.
* @param onText - Optional text output captured by the caller's execution environment.
* @returns App objects retaining tool approval, logging, image delivery and screenshot metadata. Rejections propagate without retries.
* @returns Application and window objects using the current tool binding. Rejections propagate without retries.
*/
export function connectCua(getTool: () => CuaToolBinding, onText?: (text: string) => void | Promise<void>): CuaApi {
return createCuaApi(operation => getTool()(operation), {
@@ -131,6 +131,8 @@ export const ScreenshotResult = NativeScreenshotResult.extend({
/** Permissions are reported without opening system permission dialogs. */
export const CapabilitiesResult = z.object({
platform: z.string(), permissions: z.object({ accessibility: z.boolean(), screenRecording: z.boolean() }),
/** Whether the provider supports application-level accessibility observations. */
appLevelAccessibility: z.boolean(),
sessionLocked: z.boolean().nullable(),
methods: z.array(z.string()), coordinateSpace: z.literal('screenshot-pixels'), browser: z.literal(false),
})
+41 -49
View File
@@ -34,7 +34,7 @@ export interface CuaObservationSink {
}
/** Observations emit by default. Returned values are unaffected by this option. */
export interface CuaObservationOptions { emit?: boolean }
/** Full observations of the requested content; each read replaces this surface's current snapshot. */
/** Full observations of the requested content; each read replaces the application's current snapshot. */
export interface CuaAxOptions extends CuaObservationOptions {
/** Reserved compatibility option; observations are never diffs. */
disableDiffing?: boolean
@@ -85,12 +85,8 @@ export type CuaPasteOptions = Pick<CuaParameters<'paste'>, 'format'>
export type CuaSelectTextOptions = Pick<CuaParameters<'selectText'>, 'prefix' | 'suffix' | 'selectionType'>
/** Positive deltas reveal content above/left; each axis is an integer from -10 to 10. */
export type CuaScrollOptions = Pick<CuaParameters<'scroll'>, 'x' | 'y' | 'deltaLinesX' | 'deltaLinesY'> & CuaCoordinateOptions
/** An observed AX tree and independently captured image; capture is not atomic. */
export interface CuaObservation { state: string; screenshot: CuaScreenshot }
/**
* App-wide or explicitly scoped window observations and actions.
* Capture and pointer delivery reject known AX/display frame conflicts with WINDOW_CHANGED.
* Coordinate input also rejects resizing since the latest successful AX read or screenshot.
* Accessibility observations and element operations shared by an application and its windows.
* Advertised AX actions remain available; failed actions are not retried through pointer input.
*/
export interface CuaSurface {
@@ -100,32 +96,21 @@ export interface CuaSurface {
*/
show(): Promise<CuaShowResult>
/**
* Read fresh AX text, with menu contents collapsed by default.
* Read the full accessibility tree, with menu contents collapsed by default.
* The returned tree becomes the application's current snapshot; use element ids from its latest successful read.
* @param options - emission, menu expansion and field detail options.
* @returns AX text with element ids.
* @returns Full accessibility tree text with element ids, subject to the provider's traversal limits.
*/
getAXState(options?: CuaAxOptions): Promise<string>
/**
* Capture the selected window.
* @param options - emission options.
* @returns encoded model-request image with its format, pixel dimensions and window; use mimeType when saving or declaring the image.
*/
getScreenshot(options?: CuaObservationOptions): Promise<CuaScreenshot>
/**
* Read AX then capture.
* @param options - observation options.
* @returns both non-atomic observations.
*/
getAXStateAndScreenshot(options?: CuaAxOptions): Promise<CuaObservation>
getAccessibilityTree(options?: CuaAxOptions): Promise<string>
/**
* List the actions currently advertised by an element, including attribute-setting operations.
* @param elementId - Element id from this surface's latest observation.
* @param elementId - Element id from the application's current snapshot.
* @returns Provider-defined action names; empty when the element advertises none.
*/
listActions(elementId: CuaElementId): Promise<string[]>
/**
* Perform an element action, including attribute-setting operations.
* @param elementId - Element id from this surface's latest observation.
* @param elementId - Element id from the application's current snapshot.
* @param action - Exact action name from listActions.
* @param argument - Direct action argument; omit or pass null for a no-argument action, or pass the value itself for attribute setting.
* @returns Whether the provider successfully executed the requested operation.
@@ -134,10 +119,39 @@ export interface CuaSurface {
/**
* Invoke the element's default action.
* If the element's type does not support a default action, the result's dispatched field is false.
* @param elementId - Element id from this surface's latest observation.
* @param elementId - Element id from the application's current snapshot.
* @returns Whether the provider successfully executed the requested operation.
*/
invoke(elementId: CuaElementId): Promise<CuaActionResult>
/**
* Set an observed control value.
* @param elementId - Element id from the application's current snapshot.
* @param value - Replacement value as JSON; supported values are defined by the platform provider.
* @returns dispatch evidence.
*/
setValue(elementId: CuaElementId, value: JsonValue): Promise<CuaActionResult>
/**
* Select one literal text occurrence through writable AXSelectedTextRange and verify its range.
* @param elementId - Element id from the application's current snapshot.
* @param text - Nonempty literal text; multiple matches require adjacent context.
* @param options - Adjacent prefix/suffix and selection or cursor placement.
* @returns Dispatch evidence; rejects unsupported, ambiguous or changed text without keyboard fallback.
*/
selectText(elementId: CuaElementId, text: string, options?: CuaSelectTextOptions): Promise<CuaActionResult>
}
/**
* An explicit window with screenshot and pointer input operations.
* Capture and pointer delivery reject known AX/display frame conflicts with WINDOW_CHANGED.
* Coordinate input also rejects resizing since the latest successful AX read or screenshot.
*/
export interface CuaWindow extends CuaSurface {
readonly windowId: number
/**
* Capture the selected window.
* @param options - emission options.
* @returns encoded model-request image with its format, pixel dimensions and window; use mimeType when saving or declaring the image.
*/
getScreenshot(options?: CuaObservationOptions): Promise<CuaScreenshot>
/**
* Reveal advertised AX alternate UI, otherwise move the system cursor without pressing a button.
* The system-pointer route requires the app to be foreground and the point unobscured; never activates or raises it.
@@ -156,21 +170,6 @@ export interface CuaSurface {
* @returns dispatch evidence.
*/
click(target: CuaElementId | readonly [number, number], options?: CuaClickOptions): Promise<CuaActionResult>
/**
* Set an observed control value.
* @param elementId - element id from this surface's latest observation.
* @param value - Replacement value as JSON; supported values are defined by the platform provider.
* @returns dispatch evidence.
*/
setValue(elementId: CuaElementId, value: JsonValue): Promise<CuaActionResult>
/**
* Select one literal text occurrence through writable AXSelectedTextRange and verify its range.
* @param elementId - Text control element id from this surface's latest observation.
* @param text - Nonempty literal text; multiple matches require adjacent context.
* @param options - Adjacent prefix/suffix and selection or cursor placement.
* @returns Dispatch evidence; rejects unsupported, ambiguous or changed text without keyboard fallback.
*/
selectText(elementId: CuaElementId, text: string, options?: CuaSelectTextOptions): Promise<CuaActionResult>
/**
* Scroll at a screenshot-local point or integer ratio.
* @param options - point, coordinate mode and line deltas; at least one delta must be nonzero.
@@ -186,14 +185,7 @@ export interface CuaSurface {
*/
drag(from: readonly [number, number], to: readonly [number, number], options?: CuaDragOptions): Promise<CuaActionResult>
}
/** One explicit window; keyboard operations remain app-scoped because they follow focus. */
export interface CuaWindow extends CuaSurface { readonly windowId: number }
/**
* Native app handle supporting direct observations and actions without window enumeration.
* Screenshots and coordinates reuse the latest AX observation's focused on-screen,
* nonzero-size window, or require exactly one such window. Read fresh AX to follow
* focus changes; explicit window objects retain their selected target.
*/
/** Application-level accessibility, window discovery, keyboard input and clipboard operations. */
export interface CuaApp extends CuaSurface {
readonly pid: number
readonly appHandle: CuaAppHandle
@@ -211,7 +203,7 @@ export interface CuaApp extends CuaSurface {
/**
* Observe a specific window.
* @param windowId - ID obtained from getWindows.
* @returns a reusable window object with its initial AX state retained silently; getAXState emits it.
* @returns A window object with its initial accessibility tree read silently; this updates the application's current snapshot.
*/
getWindow(windowId: number): Promise<CuaWindow>
/**
@@ -265,7 +257,7 @@ export interface CuaApi {
/**
* Select an app, obtain consent, launch it in the background if necessary, and silently retain initial AX state.
* @param selector - Exact name, bundle ID, absolute application path or running PID string; ambiguous names reject.
* @returns a handle tied to this process lifetime.
* @returns An application handle tied to this process lifetime; its initial tree read becomes the current snapshot.
*/
getApp(selector: string): Promise<CuaApp>
}
+12 -12
View File
@@ -1,6 +1,6 @@
/** Window facade using the same observed surface operations as the application facade. */
import { z } from 'zod'
import { createCuaSurface, keyChord } from './api.ts'
import { createCuaObjects, keyChord } from './api.ts'
import { ActionResult, CapabilitiesResult } from './protocol.ts'
import { LaunchEntriesResult, LaunchResult, SelectedWindowsResult, WindowDiscoveryResult, WindowRequest } from './window-protocol.ts'
import type { CuaAppHandle, CuaObservationOptions, CuaObservationSink } from './types.ts'
@@ -43,23 +43,23 @@ export function createCuaWindowApi(dispatch: CuaWindowDispatch, sink?: CuaObserv
const { windowId, pid } = selectedWindow
const windowHandle = selectedWindow.windowHandle as CuaWindowHandle
const appHandle = selectedWindow.windowHandle as CuaAppHandle
const surface = createCuaSurface(async (operation) => {
const params = z.record(z.string(), z.json()).parse(operation.params)
delete params.appHandle
delete params.windowId
const result = await call(operation.method, { ...params, windowHandle })
if (operation.method !== 'getState') return result
const state = z.record(z.string(), z.json()).parse(result)
return { ...state, appHandle }
}, sink, appHandle, pid, windowId)
const window: CuaDesktopWindow = {
...surface, windowId, windowHandle,
...createCuaObjects(async (operation) => {
const params = z.record(z.string(), z.json()).parse(operation.params)
delete params.appHandle
delete params.windowId
const result = await call(operation.method, { ...params, windowHandle })
if (operation.method !== 'getState') return result
const state = z.record(z.string(), z.json()).parse(result)
return { ...state, appHandle }
}, sink).createWindow(appHandle, pid, windowId),
windowHandle,
async activate() { return ActionResult.parse(await call('activate', { windowHandle })) },
async typeText(text) { return ActionResult.parse(await call('typeText', { windowHandle, text })) },
async paste(text, options) { return ActionResult.parse(await call('paste', { windowHandle, text, ...options })) },
async pressKey(key) { return ActionResult.parse(await call('pressKey', { windowHandle, ...keyChord(key) })) },
}
await window.getAXState({ emit: false })
await window.getAccessibilityTree({ emit: false })
windows.push(window)
}
return windows
@@ -2,7 +2,7 @@
import type { Branded } from '@deepseek-ai/dsh-brand'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import type { z } from 'zod'
import type { CuaActionResult, CuaCapabilities, CuaObservationOptions, CuaPasteOptions, CuaSurface, CuaWindowInfo } from './types.ts'
import type { CuaActionResult, CuaCapabilities, CuaObservationOptions, CuaPasteOptions, CuaWindow, CuaWindowInfo } from './types.ts'
import type { LaunchEntriesResult, WindowRequest } from './window-protocol.ts'
/** Connection-owned grant for exactly one observed window lifetime. */
@@ -14,7 +14,7 @@ export type CuaWindowDispatch = (operation: CuaWindowOperation) => Promise<JsonV
/** Discovered launch entry; its identifier is accepted by launch, not by window selection. */
export type CuaLaunchEntry = z.output<typeof LaunchEntriesResult>[number]
/** Selected window; keyboard and clipboard operations require this window in front. */
export interface CuaDesktopWindow extends CuaSurface {
export interface CuaDesktopWindow extends CuaWindow {
windowId: number
windowHandle: CuaWindowHandle
/** Bring this window forward. @returns Dispatch evidence, not application success. */
@@ -32,7 +32,7 @@ export interface CuaWindowApi {
getCapabilities(options?: CuaObservationOptions): Promise<CuaCapabilities>
/** Discover top-level windows without reading AX. @param options Output options. @returns Window metadata. */
listWindows(options?: CuaObservationOptions): Promise<CuaWindowInfo[]>
/** Authorize selected windows and read their initial AX silently. A failure may leave earlier grants in place.
/** Authorize selected windows and read their initial accessibility trees silently. A failure may leave earlier grants in place.
* @param ids IDs from listWindows; duplicates reject. @returns Windows in requested order.
*/
getWindows(ids: readonly number[]): Promise<CuaDesktopWindow[]>
+207 -146
View File
@@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest'
import { brandString } from '@deepseek-ai/dsh-brand'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import { createCuaApi } from '../src/api.ts'
import type { CuaApp, CuaElementId, CuaOperation, CuaScreenshot } from '../src/types.ts'
import type { CuaApp, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts'
/** Element ids are opaque strings; these fixtures use the macOS decimal form. */
const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
@@ -13,7 +13,7 @@ const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG
const bounds = { x: -640, y: 100, width: 640, height: 480 }
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor' }
const delivered = { dispatched: true }
const capabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
const capabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
function windowInfo(windowId: number) {
return { windowId, pid: 17, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null }
@@ -77,9 +77,10 @@ describe('application discovery', () => {
const app = await f.api.getApp('Editor')
expect(app.pid).toBe(17)
expect(app.appHandle).toBe('app-17')
expect(await app.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
const window = await app.getWindow(23)
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
expect(f.dispatch.mock.calls.map(([operation]) => operation.method)).toEqual([
'getCapabilities', 'listApps', 'getApp', 'getState', 'screenshot',
'getCapabilities', 'listApps', 'getApp', 'getState', 'getState', 'screenshot',
])
expect(await app.getWindows()).toEqual([windowInfo(23)])
expect(f.sink.text).not.toHaveBeenCalled()
@@ -154,62 +155,63 @@ describe('hover across observations', () => {
it('moves to an AX element then into a popup without inserting clicks or restoring a point', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
f.dispatch.mockClear()
expect(await app.hover(element('0'))).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
await app.getAXStateAndScreenshot({ emit: false })
expect(await app.hover([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
expect(await window.hover(element('0'))).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
await window.getAccessibilityTree({ emit: false })
await window.getScreenshot({ emit: false })
expect(await window.hover([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } },
{ method: 'getState', params: { appHandle: 'app-17' } },
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0' } },
{ method: 'getState', params: { appHandle: 'app-17', windowId: 23 } },
{ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } },
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 30, y: 40 } },
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 23, x: 30, y: 40 } },
])
const window = await app.getWindow(24)
await window.hover([5, 6])
const other = await app.getWindow(24)
await other.hover([5, 6])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'hover', params: {
appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 24, x: 5, y: 6,
appHandle: 'app-17', snapshotId: 'snapshot-4', windowId: 24, x: 5, y: 6,
} })
f.dispatch.mockRejectedValueOnce(new Error('WINDOW_CHANGED'))
await expect(window.hover([5, 6])).rejects.toThrow('WINDOW_CHANGED')
await expect(other.hover([5, 6])).rejects.toThrow('WINDOW_CHANGED')
expect(f.dispatch).toHaveBeenCalledTimes(7)
})
})
describe('pointer buttons and modifiers', () => {
it.each([0, 800])('preserves %d ms for app and window clicks at elements and coordinates', async (holdMs) => {
it.each([0, 800])('preserves %d ms for window clicks at elements and coordinates', async (holdMs) => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
for (const surface of [app, window]) {
for (const target of [element('0'), [10, 20] as const]) {
await surface.click(target, { holdMs, mouseButton: 'right' })
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ method: 'click', params: { holdMs, mouseButton: 'right' } })
f.dispatch.mockClear()
await expect(surface.click(target, { holdMs: -1 })).rejects.toThrow('holdMs')
expect(f.dispatch).not.toHaveBeenCalled()
}
for (const target of [element('0'), [10, 20] as const]) {
await window.click(target, { holdMs, mouseButton: 'right' })
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ method: 'click', params: { holdMs, mouseButton: 'right' } })
f.dispatch.mockClear()
await expect(window.click(target, { holdMs: -1 })).rejects.toThrow('holdMs')
expect(f.dispatch).not.toHaveBeenCalled()
}
})
it('retains middle-button and modifier intent through click and drag dispatch', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
await app.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] })
const window = await app.getWindow(23)
await window.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', mouseButton: 'middle', modifiers: ['shift'],
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', mouseButton: 'middle', modifiers: ['shift'],
} })
await app.click([10, 20], { modifiers: ['command'] })
await window.click([10, 20], { modifiers: ['command'] })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20,
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20,
mouseButton: 'left', clickCount: 1, modifiers: ['command'],
} })
await app.drag([10, 20], [30, 40], { mouseButton: 'middle', modifiers: ['shift', 'option'] })
await window.drag([10, 20], [30, 40], { mouseButton: 'middle', modifiers: ['shift', 'option'] })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'drag', params: {
appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, fromX: 10, fromY: 20, toX: 30, toY: 40,
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', fromX: 10, fromY: 20, toX: 30, toY: 40,
mouseButton: 'middle', modifiers: ['shift', 'option'],
} })
const count = f.dispatch.mock.calls.length
await expect(app.click(element('0'), { mouseButton: 'middle', clickCount: 2 })).rejects.toThrow('Only left double-click')
await expect(window.click(element('0'), { mouseButton: 'middle', clickCount: 2 })).rejects.toThrow('Only left double-click')
expect(f.dispatch).toHaveBeenCalledTimes(count)
})
})
@@ -307,30 +309,87 @@ describe('element default invocation', () => {
})
describe('application and window observations', () => {
it('keeps the latest element id scope separate for an app and each selected window', async () => {
it('shares the latest tree read across an application and each selected window', async () => {
const f = fixture()
f.setWindows([23, 24])
const app = await f.api.getApp('Editor')
expect(await app.getWindows({ emit: false })).toEqual([windowInfo(23), windowInfo(24)])
const first = await app.getWindow(23)
const second = await app.getWindow(24)
await app.click(element('0'))
await first.setValue(element('1'), 'First document')
await second.click(element('2'))
await app.setValue(element('1'), 'First document')
await first.click(element('2'))
await second.click(element('3'))
expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } },
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '1', value: 'First document' } },
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '1', value: 'First document' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3' } },
])
expect(first.windowId).toBe(23)
expect(second.windowId).toBe(24)
expect(f.sink.text.mock.calls.map(([text]) => text)).toEqual([])
await expect(second.getAXState({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4')
await expect(second.getAccessibilityTree({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4')
await second.click(element('3'))
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3' } })
await app.getAccessibilityTree({ emit: false })
await first.invoke(element('4'))
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4' } })
expect(f.sink.text).not.toHaveBeenCalled()
})
it('keeps separate snapshots for different application handles', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockResolvedValueOnce({ appHandle: 'app-18', pid: 18, name: 'Utility' })
f.dispatch.mockResolvedValueOnce(state({ appHandle: 'app-18', pid: 18, snapshotId: 'other', windows: [] }))
const other = await f.api.getApp('Utility')
await app.setValue(element('0'), 'Editor')
await other.setValue(element('0'), 'Utility')
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', value: 'Editor' } },
{ method: 'setValue', params: { appHandle: 'app-18', snapshotId: 'other', elementId: '0', value: 'Utility' } },
])
})
it('shares one current snapshot between repeated handles for the same application', async () => {
const f = fixture()
const first = await f.api.getApp('Editor')
const second = await f.api.getApp('Editor')
expect(second.appHandle).toBe(first.appHandle)
await first.setValue(element('1'), 'Shared')
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'setValue', params: {
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '1', value: 'Shared',
} })
})
it('exposes screenshots and pointer input only on explicit windows', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
expect(app).toHaveProperty('getAccessibilityTree')
for (const method of ['getScreenshot', 'click', 'hover', 'scroll', 'drag']) expect(app).not.toHaveProperty(method)
const window = await app.getWindow(23)
for (const method of ['getScreenshot', 'click', 'hover', 'scroll', 'drag']) expect(window).toHaveProperty(method)
})
it('attaches the shared snapshot to coordinates only within its window scope', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const first = await app.getWindow(23)
await first.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
const second = await app.getWindow(24)
await first.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
await second.click([10, 20])
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-3', windowId: 24 } })
await app.getAccessibilityTree({ emit: false })
await first.click([10, 20])
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-4', windowId: 23 } })
})
it('keeps a validated AXless window usable for screenshots and coordinate input', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
@@ -357,17 +416,7 @@ describe('application and window observations', () => {
expect(f.dispatch.mock.calls.slice(calls).map(([operation]) => operation.method)).toEqual(['getState'])
})
it.each([{ ids: [] }, { ids: [23, 24] }])('requires an explicit window when discovery yields $ids', async ({ ids }) => {
const f = fixture()
f.setWindows(ids)
const app = await f.api.getApp('Editor')
const calls = f.dispatch.mock.calls.length
await expect(app.click([1, 2])).rejects.toThrow('Select a window')
await expect(app.getScreenshot()).rejects.toThrow('Select a window')
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
it('selects the one visible nonzero window while preserving menu and hidden windows in discovery', async () => {
it('captures and clicks the explicitly selected window among several', async () => {
const f = fixture()
const windows = [
windowInfo(23),
@@ -378,46 +427,28 @@ describe('application and window observations', () => {
f.setWindowRecords(windows)
const app = await f.api.getApp('Editor')
expect(await app.getWindows({ emit: false })).toEqual(windows)
await app.getScreenshot()
const window = await app.getWindow(23)
await window.getScreenshot()
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } })
await app.click([10, 20])
await window.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
const hidden = await app.getWindow(25)
await hidden.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 25, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
appHandle: 'app-17', windowId: 25, snapshotId: 'snapshot-3', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
})
it('requires explicit selection when every window is hidden or has zero size', async () => {
it('captures a window without reading its accessibility tree', async () => {
const f = fixture()
f.setWindowRecords([
{ ...windowInfo(24), bounds: { ...bounds, width: 0, height: 0 } },
{ ...windowInfo(25), isOnScreen: false },
])
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const calls = f.dispatch.mock.calls.length
await expect(app.getScreenshot()).rejects.toThrow('Select a window')
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
it('captures the focused application window when several visible windows exist', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockResolvedValueOnce(state({ focusedWindowId: 24, windows: [windowInfo(23), windowInfo(24)] }))
await app.getAXState()
await app.getScreenshot()
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 24 } })
})
it('does not select a focused window absent from the application observation', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockResolvedValueOnce(state({ focusedWindowId: 99, windows: [windowInfo(23), windowInfo(24)] }))
await app.getAXState()
await expect(app.getScreenshot()).rejects.toThrow('Select a window')
await window.getScreenshot()
expect(f.dispatch).toHaveBeenCalledTimes(calls + 1)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } })
})
const identityMismatches: Record<string, JsonValue>[] = [
@@ -427,9 +458,9 @@ describe('application and window observations', () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockResolvedValueOnce(state(identity))
await expect(app.getAXState()).rejects.toThrow('another application or window')
await expect(app.getAccessibilityTree()).rejects.toThrow('another application or window')
const calls = f.dispatch.mock.calls.length
await expect(app.click(element('0'))).rejects.toThrow('Read getAXState')
await expect(app.setValue(element('0'), 'stale')).rejects.toThrow('Read getAccessibilityTree')
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
@@ -438,22 +469,40 @@ describe('application and window observations', () => {
const app = await f.api.getApp('Editor')
const otherWindow = { ...windowInfo(23), pid: 18 }
f.dispatch.mockResolvedValueOnce(state({ windows: [otherWindow] }))
await expect(app.getAXState()).rejects.toThrow('windows belong to another application')
await expect(app.getAccessibilityTree()).rejects.toThrow('windows belong to another application')
f.dispatch.mockResolvedValueOnce([otherWindow])
await expect(app.getWindows()).rejects.toThrow('windows belong to another application')
expect(f.sink.text).not.toHaveBeenCalled()
})
it('clears a failed refresh and obtains new state before a later screenshot', async () => {
it('accepts coordinate input without a current snapshot', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
f.dispatch.mockRejectedValueOnce(new Error('Native app is busy'))
await expect(window.getAccessibilityTree()).rejects.toThrow('Native app is busy')
await window.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
})
it('clears a failed refresh and requires a new read before element operations', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const failure = new Error('Native app is busy')
f.dispatch.mockRejectedValueOnce(failure)
await expect(app.getAXState()).rejects.toBe(failure)
await expect(app.setValue(element('0'), 'stale')).rejects.toThrow('Read getAXState')
await expect(app.getAccessibilityTree()).rejects.toBe(failure)
await expect(window.setValue(element('0'), 'stale')).rejects.toThrow('Read getAccessibilityTree')
await window.getAccessibilityTree({ emit: false })
f.dispatch.mockRejectedValueOnce(failure)
await expect(window.getAccessibilityTree()).rejects.toBe(failure)
await expect(app.invoke(element('0'))).rejects.toThrow('Read getAccessibilityTree')
f.sink.text.mockClear()
await app.getScreenshot()
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation.method)).toEqual(['getState', 'screenshot'])
const calls = f.dispatch.mock.calls.length
await window.getScreenshot()
expect(f.dispatch.mock.calls.slice(calls).map(([operation]) => operation.method)).toEqual(['screenshot'])
expect(f.sink.text).not.toHaveBeenCalled()
expect(f.sink.image).toHaveBeenCalledTimes(1)
})
@@ -466,13 +515,15 @@ describe('screenshots', () => {
const app = await f.api.getApp('Editor')
const window = await app.getWindow(24)
f.sink.text.mockClear()
const observation = await window.getAXStateAndScreenshot()
expect(observation.state).toBe('Tree 3')
expect(observation.screenshot).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 24 })
const tree = await window.getAccessibilityTree()
const image = await window.getScreenshot()
expect(tree).toBe('Tree 3')
expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 24 })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 24 } })
expect(f.sink.text).toHaveBeenCalledWith('Tree 3')
expect(f.sink.image).toHaveBeenCalledWith(observation.screenshot)
await window.getAXStateAndScreenshot({ emit: false })
expect(f.sink.image).toHaveBeenCalledWith(image)
await window.getAccessibilityTree({ emit: false })
await window.getScreenshot({ emit: false })
expect(f.sink.text).toHaveBeenCalledTimes(1)
expect(f.sink.image).toHaveBeenCalledTimes(1)
})
@@ -480,17 +531,19 @@ describe('screenshots', () => {
it('rejects a screenshot captured from another window before emitting it', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
f.dispatch.mockResolvedValueOnce(screenshot({ windowId: 24 }))
await expect(app.getScreenshot()).rejects.toThrow('another window')
await expect(window.getScreenshot()).rejects.toThrow('another window')
expect(f.sink.image).not.toHaveBeenCalled()
})
it.each(['image/jpeg', 'image/webp', 'image/gif'])('preserves Host-admitted %s bytes and metadata in the return value and image sink', async (mimeType) => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const data = Buffer.from('admitted request image').toString('base64')
f.dispatch.mockResolvedValueOnce(screenshot({ mimeType, data, pixelWidth: 320, pixelHeight: 240 }))
const image = await app.getScreenshot()
const image = await window.getScreenshot()
expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(data, 'base64')), mimeType, pixelWidth: 320, pixelHeight: 240, windowId: 23 })
expect(f.sink.image).toHaveBeenCalledExactlyOnceWith(image)
})
@@ -501,21 +554,22 @@ describe('input actions', () => {
it('passes explicit relative mode to every coordinate gesture and rejects it on element ids', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
await app.click([500, 250], { relative: true })
await app.hover([500, 250], { relative: true })
await app.drag([0, 0], [999, 999], { relative: true })
await app.scroll({ x: 500, y: 250, relative: true, deltaLinesX: 0, deltaLinesY: -1 })
const window = await app.getWindow(23)
await window.click([500, 250], { relative: true })
await window.hover([500, 250], { relative: true })
await window.drag([0, 0], [999, 999], { relative: true })
await window.scroll({ x: 500, y: 250, relative: true, deltaLinesX: 0, deltaLinesY: -1 })
expect(f.dispatch.mock.calls.slice(-4).map(([request]) => request.params)).toEqual([
expect.objectContaining({ x: 500, y: 250, relative: true }),
expect.objectContaining({ x: 500, y: 250, relative: true }),
expect.objectContaining({ fromX: 0, toX: 999, relative: true }),
expect.objectContaining({ x: 500, y: 250, relative: true }),
])
await expect(app.click(element('1'), { relative: true })).rejects.toThrow()
await expect(app.hover(element('1'), { relative: true })).rejects.toThrow()
await expect(window.click(element('1'), { relative: true })).rejects.toThrow()
await expect(window.hover(element('1'), { relative: true })).rejects.toThrow()
})
it('keeps the selected app and window when scroll options include another target identity', async () => {
it('keeps the selected window when scroll options include another target identity', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
@@ -534,57 +588,61 @@ describe('input actions', () => {
it('dispatches window-point gestures and reports delivery without claiming the action succeeded', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
expect(await app.click([10, 20], { mouseButton: 'right' })).toEqual(delivered)
expect(await app.click([11, 21], { clickCount: 2 })).toEqual(delivered)
expect(await app.scroll({ x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 })).toEqual(delivered)
expect(await app.drag([1, 2], [30, 40])).toEqual(delivered)
const window = await app.getWindow(23)
expect(await window.click([10, 20], { mouseButton: 'right' })).toEqual(delivered)
expect(await window.click([11, 21], { clickCount: 2 })).toEqual(delivered)
expect(await window.scroll({ x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 })).toEqual(delivered)
expect(await window.drag([1, 2], [30, 40])).toEqual(delivered)
expect(f.dispatch.mock.calls.slice(-4).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'right', clickCount: 1 } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 11, y: 21, mouseButton: 'left', clickCount: 2 } },
{ method: 'scroll', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } },
{ method: 'drag', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, fromX: 1, fromY: 2, toX: 30, toY: 40 } },
{ method: 'click', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'right', clickCount: 1 } },
{ method: 'click', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 11, y: 21, mouseButton: 'left', clickCount: 2 } },
{ method: 'scroll', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } },
{ method: 'drag', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', fromX: 1, fromY: 2, toX: 30, toY: 40 } },
])
})
const invalidActions: Array<[string, (app: CuaApp) => Promise<unknown>]> = [
['right double click', app => app.click([1, 2], { mouseButton: 'right', clickCount: 2 })],
['right AX double click', app => app.click(element('0'), { mouseButton: 'right', clickCount: 2 })],
['empty hover element id', app => app.hover(element(''))],
['negative hover point', app => app.hover([-1, 0])],
['nonfinite hover point', app => app.hover([NaN, 0])],
['empty click element id', app => app.click(element(''))],
['nonfinite coordinate', app => app.click([NaN, 0])],
['negative coordinate', app => app.click([-1, 0])],
['empty value element id', app => app.setValue(element(''), 'value')],
['zero scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0, deltaLinesY: 0 })],
['oversized scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })],
['fractional scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })],
['negative drag point', app => app.drag([0, 0], [-1, 3])],
['empty chord', app => app.pressKey('')],
['missing key', app => app.pressKey('super+')],
['empty plus modifier', app => app.pressKey('++')],
['extra plus delimiter', app => app.pressKey('cmd+++')],
['unknown modifier', app => app.pressKey('unknown+a')],
type Surfaces = { app: CuaApp; window: CuaWindow }
const invalidActions: Array<[string, (surfaces: Surfaces) => Promise<unknown>]> = [
['right double click', ({ window }) => window.click([1, 2], { mouseButton: 'right', clickCount: 2 })],
['right AX double click', ({ window }) => window.click(element('0'), { mouseButton: 'right', clickCount: 2 })],
['empty hover element id', ({ window }) => window.hover(element(''))],
['negative hover point', ({ window }) => window.hover([-1, 0])],
['nonfinite hover point', ({ window }) => window.hover([NaN, 0])],
['empty click element id', ({ window }) => window.click(element(''))],
['nonfinite coordinate', ({ window }) => window.click([NaN, 0])],
['negative coordinate', ({ window }) => window.click([-1, 0])],
['empty value element id', ({ window }) => window.setValue(element(''), 'value')],
['zero scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0, deltaLinesY: 0 })],
['oversized scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })],
['fractional scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })],
['negative drag point', ({ window }) => window.drag([0, 0], [-1, 3])],
['empty chord', ({ app }) => app.pressKey('')],
['missing key', ({ app }) => app.pressKey('super+')],
['empty plus modifier', ({ app }) => app.pressKey('++')],
['extra plus delimiter', ({ app }) => app.pressKey('cmd+++')],
['unknown modifier', ({ app }) => app.pressKey('unknown+a')],
]
it.each<[string, (app: CuaApp) => Promise<unknown>]>([
['getState windowId: Too small: expected number to be >0', app => app.getWindow(0)],
['selectText text: Too small: expected string to have >=1 characters', app => app.selectText(element('0'), '')],
['hover x: Invalid input: expected number, received NaN', app => app.hover([NaN, 0])],
['hover x: Too big: expected number to be <=999', app => app.hover([1000, 0], { relative: true })],
['click relative: not accepted for these arguments', app => app.click(element('650'), { relative: true })],
it.each<[string, (surfaces: Surfaces) => Promise<unknown>]>([
['getState windowId: Too small: expected number to be >0', ({ app }) => app.getWindow(0)],
['selectText text: Too small: expected string to have >=1 characters', ({ app }) => app.selectText(element('0'), '')],
['hover x: Invalid input: expected number, received NaN', ({ window }) => window.hover([NaN, 0])],
['hover x: Too big: expected number to be <=999', ({ window }) => window.hover([1000, 0], { relative: true })],
['click relative: not accepted for these arguments', ({ window }) => window.click(element('650'), { relative: true })],
])('reports a readable validation failure: %s', async (message, run) => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
f.dispatch.mockClear()
await expect(run(app)).rejects.toMatchObject({ name: 'Error', message })
await expect(run({ app, window })).rejects.toMatchObject({ name: 'Error', message })
expect(f.dispatch).not.toHaveBeenCalled()
})
it.each(invalidActions)('rejects %s without invoking native input', async (_label, run) => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const calls = f.dispatch.mock.calls.length
await expect(run(app)).rejects.toThrow()
await expect(run({ app, window })).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
@@ -621,22 +679,24 @@ describe('input actions', () => {
it('retains the snapshot identity for element right and double clicks', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
await app.click(element('12'), { mouseButton: 'right' })
await app.click(element('13'), { clickCount: 2 })
const window = await app.getWindow(23)
await window.click(element('12'), { mouseButton: 'right' })
await window.click(element('13'), { clickCount: 2 })
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '12', mouseButton: 'right' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '13', clickCount: 2 } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', mouseButton: 'right' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', clickCount: 2 } },
])
})
it('dispatches an opaque provider element id unchanged', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
await app.click(element('42.12345'))
await app.hover(element('42.12345'))
const window = await app.getWindow(23)
await window.click(element('42.12345'))
await window.hover(element('42.12345'))
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345' } },
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } },
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } },
])
})
@@ -651,13 +711,14 @@ describe('input actions', () => {
it('propagates native failures once and rejects malformed delivery claims', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const failure = new Error('Application was restarted; select it again')
f.dispatch.mockRejectedValueOnce(failure)
await expect(app.typeText('hello')).rejects.toBe(failure)
expect(f.dispatch).toHaveBeenCalledTimes(3)
f.dispatch.mockResolvedValueOnce({ dispatched: 'yes' })
await expect(app.click(element('0'))).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(4)
f.dispatch.mockResolvedValueOnce({ dispatched: 'yes' })
await expect(window.click(element('0'))).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(5)
})
it('rejects invalid selection inputs before dispatch', async () => {
@@ -674,8 +735,8 @@ describe('input actions', () => {
it('expands menu contents and details only for the requested observation', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
await app.getAXState({ includeMenus: true, details: true, emit: false })
await app.getAccessibilityTree({ includeMenus: true, details: true, emit: false })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'getState', params: { appHandle: selected.appHandle, includeMenus: true, details: true } })
await app.getAXState({ includeMenus: false, details: false })
await app.getAccessibilityTree({ includeMenus: false, details: false })
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'getState', params: { appHandle: selected.appHandle, includeMenus: false, details: false } })
})
+13 -7
View File
@@ -19,20 +19,25 @@ const snapshot = {
}
it('uses the current tool binding when a retained app is called by another execution', async () => {
const first = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => operation.method === 'getApp' ? selected : snapshot)
const first = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => {
if (operation.method === 'getApp') return selected
if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null }
return snapshot
})
let current: CuaToolBinding = first
const text = vi.fn()
const cua = connectCua(() => current, text)
const app = await cua.getApp('Editor')
const window = await app.getWindow(23)
const second = vi.fn(async (): Promise<JsonValue> => ({ dispatched: true }))
current = second
await expect(app.click(element('0'))).resolves.toEqual({ dispatched: true })
expect(first).toHaveBeenCalledTimes(2)
await expect(window.click(element('0'))).resolves.toEqual({ dispatched: true })
expect(first).toHaveBeenCalledTimes(3)
expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } })
expect(text).not.toHaveBeenCalled()
current = first
await app.getAXState()
await app.getAccessibilityTree()
expect(text).toHaveBeenCalledWith('0 button')
})
@@ -57,16 +62,17 @@ it.each([null, 'text only', { content: [{ type: 'text', text: '{}' }] }])('rejec
it('returns screenshot bytes and metadata with one image-bearing tool call and no image text output', async () => {
const tool = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => {
if (operation.method === 'getApp') return selected
if (operation.method === 'getState') return snapshot
if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null }
return {
mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 23, bounds, coordinateSpace: 'screenshot-pixels',
}
})
const text = vi.fn()
const app = await connectCua(() => tool, text).getApp('Editor')
const window = await app.getWindow(23)
text.mockClear()
expect(await app.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getApp', 'getState', 'screenshot'])
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getApp', 'getState', 'getState', 'screenshot'])
expect(text).not.toHaveBeenCalled()
})
@@ -1,7 +1,7 @@
/** Model-authored operation JSON cannot supply host configuration or bypass retained identities. */
import { expect, it } from 'vitest'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import { Request, ScreenshotResult } from '../src/protocol.ts'
import { CapabilitiesResult, Request, ScreenshotResult } from '../src/protocol.ts'
const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [
{ reason: 'raw process ID instead of a selected app handle', request: { method: 'typeText', params: { pid: 17, text: 'hello' } } },
@@ -155,3 +155,13 @@ it.each(['click', 'hover', 'scroll', 'drag'])('validates 0–999 integer coordin
expect(Request.safeParse({ method, params: { ...params, relative: false, [field]: 1000 } }).success).toBe(true)
}
})
it.each([true, false])('requires the reported application-level accessibility capability: %s', (appLevelAccessibility) => {
const capabilities = {
platform: 'macos', permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility,
sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false,
}
expect(CapabilitiesResult.parse(capabilities)).toEqual(capabilities)
const { appLevelAccessibility: _omitted, ...missing } = capabilities
expect(CapabilitiesResult.safeParse(missing).success).toBe(false)
})
@@ -11,6 +11,7 @@ import type { CuaWindowOperation } from '../src/window-types.ts'
const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
const metadata = { windowId: 7, pid: 10, appName: 'Editor', title: 'Document', bounds: { x: -100, y: 0, width: 800, height: 600 }, isOnScreen: true, onCurrentSpace: null }
const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC'
function fixture() {
const text = vi.fn()
const dispatch = vi.fn(async (request: CuaWindowOperation): Promise<JsonValue> => {
@@ -20,6 +21,8 @@ function fixture() {
case 'getState': return { windowHandle: 'window-7', windowId: 7, pid: 10, snapshotId: 'state-1', tree: 'AX window', windows: [metadata], truncated: false }
case 'findApps': return [{ id: 'shell-entry', displayName: 'Settings', kind: 'shell' }]
case 'listActions': return ['AXPress']
case 'hover': return { dispatched: true, hoverRoute: 'accessibility', systemPointerMoved: false }
case 'screenshot': return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 7, bounds: metadata.bounds, coordinateSpace: 'screenshot-pixels' }
default: return { dispatched: true }
}
})
@@ -39,9 +42,15 @@ it('binds all observation and keyboard operations to a selected window and suppr
expect(window).toBeDefined()
if (!window) throw new Error('Missing selected window')
expect(f.text).not.toHaveBeenCalled()
await window.getAXState()
expect(f.dispatch.mock.calls[1]?.[0]).toEqual({ method: 'getState', params: { windowHandle: 'window-7' } })
await window.getAccessibilityTree()
expect(f.text).toHaveBeenCalledWith('AX window')
await window.click(element('0'))
await window.click([1, 2])
await window.hover(element('42.12345'))
await window.scroll({ x: 1, y: 2, deltaLinesX: 0, deltaLinesY: 1 })
await window.drag([1, 2], [3, 4])
expect(await window.getScreenshot()).toMatchObject({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 7 })
await window.activate()
await window.typeText('hello')
await window.paste('world')
@@ -116,7 +125,7 @@ it('supports discovery without a sink and suppresses requested emissions', async
await f.api.listWindows({ emit: false })
expect(f.text).not.toHaveBeenCalled()
await createCuaWindowApi(f.dispatch).listWindows()
const capabilities = { platform: 'windows', permissions: { accessibility: true, screenRecording: true }, sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false }
const capabilities = { platform: 'windows', permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: false, sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false }
f.dispatch.mockResolvedValueOnce(capabilities)
expect(await f.api.getCapabilities()).toEqual(capabilities)
})
@@ -131,7 +131,7 @@ function createApplicationState(socket, record, application, captureBroker, conn
const state = JSON.parse(readFileSync(new URL('./permissions.json', import.meta.url), 'utf8'))
return { platform: 'darwin', ...state, helper: { pid: process.pid }, parent: { pid: process.ppid } }
}
if (method === 'getCapabilities') return { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: true }, methods, coordinateSpace: 'window-points', browser: false }
if (method === 'getCapabilities') return { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: true, methods, coordinateSpace: 'window-points', browser: false }
if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId }]
if (method === 'getApp') {
if (params.selector === 'Denied Fixture' || params.selector === 'Revocable Fixture') {