refactor(cua): separate window operations from accessibility surfaces
This commit is contained in:
1 parent
0c4eb10d94
commit
bfd5e9fccc
15 files changed
+428
-346
No files matched your search
@@ -193,6 +193,7 @@ actor AXWorker {
|
||||
if method == "getCapabilities" {
|
||||
return .result(.object(["platform": .string("macos"), "permissions": .object([
|
||||
"accessibility": .bool(AXIsProcessTrusted()), "screenRecording": .bool(CGPreflightScreenCaptureAccess())]),
|
||||
"appLevelAccessibility": .bool(true),
|
||||
"sessionLocked": locked.map(JSONValue.bool) ?? .null,
|
||||
"methods": .array(NativeProtocol.modelMethods.map(JSONValue.string)), "coordinateSpace": .string("window-points"), "browser": .bool(false)]))
|
||||
}
|
||||
|
||||
@@ -173,7 +173,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
|
||||
})
|
||||
const retainedApp = await objectApi(agent).getApp('test.fixture')
|
||||
const retainedWindow = await retainedApp.getWindow(7)
|
||||
expect(await retainedWindow.getAXState({ emit: false })).toContain('Count: 1')
|
||||
expect(await retainedWindow.getAccessibilityTree({ emit: false })).toContain('Count: 1')
|
||||
const peer = await context.agents.create({ sessionId: SessionId('native-loader-peer'), agentOptions: { provider: 'native-fixture', model: 'vision' } })
|
||||
const peerApp = await objectApi(peer.agent).getApp('test.fixture')
|
||||
const peerWindow = await peerApp.getWindow(7)
|
||||
@@ -182,7 +182,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
|
||||
expect(screenshot).toMatchObject({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 7 })
|
||||
expect(Buffer.from(screenshot.bytes).toString('base64')).toBe(png)
|
||||
await peerWindow.click(element('0'))
|
||||
expect(await retainedWindow.getAXState({ emit: false })).toContain('Count: 2')
|
||||
expect(await retainedWindow.getAccessibilityTree({ emit: false })).toContain('Count: 2')
|
||||
const sharedEvents = await readEvents()
|
||||
expect(sharedEvents.filter(event => event.event === 'start')).toHaveLength(1)
|
||||
expect(sharedEvents.filter(event => event.event === 'connect')).toHaveLength(2)
|
||||
@@ -191,7 +191,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained
|
||||
expect((await readEvents()).filter(event => event.event === 'exit')).toHaveLength(0)
|
||||
expect((await readEvents()).filter(event => event.event === 'disconnect')).toHaveLength(1)
|
||||
await peerWindow.click(element('0'))
|
||||
expect(await peerWindow.getAXState({ emit: false })).toContain('Count: 3')
|
||||
expect(await peerWindow.getAccessibilityTree({ emit: false })).toContain('Count: 3')
|
||||
await peer.dispose()
|
||||
expect((await readEvents()).filter(event => event.event === 'exit')).toHaveLength(1)
|
||||
|
||||
|
||||
@@ -84,7 +84,7 @@ it.skipIf(process.platform === 'win32' || !existsSync(bridge) || !existsSync(boo
|
||||
expect(invalid.error?.message).not.toContain('ZodError')
|
||||
expect(invalid.error?.message).not.toContain('"origin"')
|
||||
expect(calls.filter(call => call.phase === 'invalid')).toEqual([])
|
||||
const second = await run('await win.getAXState({emit:false}); await win.click("0"); await win.getAXState({emit:false});', 'second')
|
||||
const second = await run('await win.getAccessibilityTree({emit:false}); await win.click("0"); await win.getAccessibilityTree({emit:false});', 'second')
|
||||
expect(second.error).toBeUndefined()
|
||||
expect(second.value).toContain('Count: 2')
|
||||
expect(calls.filter(call => call.phase === 'second').map(call => call.method)).toEqual(['getState', 'click', 'getState'])
|
||||
|
||||
@@ -164,7 +164,7 @@ it('correlates a native request with its logged tool invocation', async () => {
|
||||
|
||||
it('reports screenshot pixel coordinates while preserving native capabilities', async () => {
|
||||
const f = await setup()
|
||||
const capabilities = { platform: 'macos', coordinateSpace: 'window-points', permissions: { accessibility: true, screenCapture: false } }
|
||||
const capabilities = { platform: 'macos', coordinateSpace: 'window-points', permissions: { accessibility: true, screenCapture: false }, appLevelAccessibility: true }
|
||||
native.request.mockResolvedValueOnce(capabilities)
|
||||
const result = await f.execute('getCapabilities')
|
||||
expect(result.isError).toBe(false)
|
||||
|
||||
@@ -68,7 +68,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
|
||||
async function emit(value: object, options?: CuaObservationOptions): Promise<void> {
|
||||
if (options?.emit !== false) await sink?.text(JSON.stringify(value))
|
||||
}
|
||||
|
||||
const objects = createCuaObjects(dispatch, sink)
|
||||
|
||||
const api: CuaApi = {
|
||||
async getCapabilities(options) {
|
||||
@@ -96,7 +96,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
|
||||
async getApp(selector) {
|
||||
const { appHandle, pid } = AppResult.parse(await call('getApp', { selector }))
|
||||
const app: CuaApp = {
|
||||
...createCuaSurface(dispatch, sink, appHandle, pid), pid, appHandle,
|
||||
...objects.createSurface(appHandle, pid), pid, appHandle,
|
||||
async activate() { return action('activate', { appHandle }) },
|
||||
async getWindows(options) {
|
||||
const windows = WindowsResult.parse(await call('listWindows', { appHandle }))
|
||||
@@ -105,8 +105,8 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
|
||||
return windows
|
||||
},
|
||||
async getWindow(windowId): Promise<CuaWindow> {
|
||||
const window = { ...createCuaSurface(dispatch, sink, appHandle, pid, windowId), windowId }
|
||||
await window.getAXState({ emit: false })
|
||||
const window = objects.createWindow(appHandle, pid, windowId)
|
||||
await window.getAccessibilityTree({ emit: false })
|
||||
return window
|
||||
},
|
||||
async typeText(text) { return action('typeText', { appHandle, text }) },
|
||||
@@ -115,7 +115,7 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
|
||||
}) },
|
||||
async pressKey(key) { return action('pressKey', { appHandle, ...keyChord(key) }) },
|
||||
}
|
||||
await app.getAXState({ emit: false })
|
||||
await app.getAccessibilityTree({ emit: false })
|
||||
return app
|
||||
},
|
||||
}
|
||||
@@ -123,127 +123,128 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind one observed surface to its authorized dispatcher.
|
||||
* Create application and window objects sharing their latest observations.
|
||||
* @param dispatch Logged operation dispatcher.
|
||||
* @param sink Optional observation output.
|
||||
* @param appHandle Native authorization handle.
|
||||
* @param pid Observed process lifetime owner.
|
||||
* @param selectedWindow Explicit window selection, or the macOS application scope.
|
||||
* @returns Observation and input methods retaining only the latest AX state.
|
||||
* @returns Surface and window constructors with observation state owned by this factory.
|
||||
*/
|
||||
export function createCuaSurface(dispatch: CuaDispatch, sink: CuaObservationSink | undefined,
|
||||
appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface {
|
||||
export function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | undefined): {
|
||||
createSurface(appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface
|
||||
createWindow(appHandle: CuaAppHandle, pid: number, windowId: number): CuaWindow
|
||||
} {
|
||||
const call = validatedCall(dispatch)
|
||||
const action = async (method: CuaOperation['method'], params: Record<string, JsonValue>) => ActionResult.parse(await call(method, params))
|
||||
let observed: ReturnType<typeof StateResult.parse> | undefined
|
||||
function snapshotParams(): Record<string, JsonValue> {
|
||||
if (!observed) throw new Error('Read getAXState before using this surface')
|
||||
return { appHandle, snapshotId: observed.snapshotId }
|
||||
const observed = new Map<CuaAppHandle, ReturnType<typeof StateResult.parse>>()
|
||||
function elementParams(appHandle: CuaAppHandle): Record<string, JsonValue> {
|
||||
const current = observed.get(appHandle)
|
||||
if (current === undefined) throw new Error('Read getAccessibilityTree before using this surface')
|
||||
return { appHandle, snapshotId: current.snapshotId }
|
||||
}
|
||||
function windowId(): number {
|
||||
if (selectedWindow !== undefined) return selectedWindow
|
||||
const windows = observed?.windows.filter(window => window.isOnScreen && window.bounds.width > 0 && window.bounds.height > 0)
|
||||
const focused = windows?.find(window => window.windowId === observed?.focusedWindowId)
|
||||
if (focused) return focused.windowId
|
||||
const first = windows?.[0]
|
||||
if (!first || windows.length !== 1) throw new Error('Select a window with app.getWindows() and app.getWindow(windowId) before this operation')
|
||||
return first.windowId
|
||||
function coordinateParams(appHandle: CuaAppHandle, windowId: number): Record<string, JsonValue> {
|
||||
const current = observed.get(appHandle)
|
||||
return {
|
||||
appHandle, windowId,
|
||||
...current !== undefined && (current.windowId === null || current.windowId === windowId)
|
||||
? { snapshotId: current.snapshotId } : {},
|
||||
}
|
||||
}
|
||||
const result: CuaSurface = {
|
||||
async show() {
|
||||
return ShowResult.parse(await call('show', { appHandle,
|
||||
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
|
||||
}))
|
||||
},
|
||||
async getAXState(options) {
|
||||
observed = undefined
|
||||
const state = StateResult.parse(await call('getState', { appHandle,
|
||||
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
|
||||
...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus },
|
||||
...options?.details === undefined ? {} : { details: options.details },
|
||||
}))
|
||||
if (state.appHandle !== appHandle || state.pid !== pid || state.windowId !== (selectedWindow ?? null)) {
|
||||
throw new Error('Native observation belongs to another application or window')
|
||||
}
|
||||
if (state.windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application')
|
||||
observed = state
|
||||
if (options?.emit !== false) await sink?.text(state.tree)
|
||||
return state.tree
|
||||
},
|
||||
async getScreenshot(options) {
|
||||
if (!observed) await result.getAXState({ emit: false })
|
||||
const target = windowId()
|
||||
const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId: target }))
|
||||
if (image.windowId !== target) throw new Error('Native screenshot belongs to another window')
|
||||
const screenshot: CuaScreenshot = {
|
||||
bytes: Uint8Array.from(Buffer.from(image.data, 'base64')),
|
||||
mimeType: image.mimeType,
|
||||
pixelWidth: image.pixelWidth,
|
||||
pixelHeight: image.pixelHeight,
|
||||
windowId: image.windowId,
|
||||
}
|
||||
if (options?.emit !== false) await sink?.image(screenshot)
|
||||
return screenshot
|
||||
},
|
||||
async getAXStateAndScreenshot(options) {
|
||||
const state = await result.getAXState(options)
|
||||
return { state, screenshot: await result.getScreenshot(options) }
|
||||
},
|
||||
async listActions(elementId) {
|
||||
return ActionNamesResult.parse(await call('listActions', { ...snapshotParams(), elementId }))
|
||||
},
|
||||
async performAction(elementId, name, argument) {
|
||||
return action('performAction', { ...snapshotParams(), elementId, action: name,
|
||||
...argument === undefined ? {} : { argument },
|
||||
})
|
||||
},
|
||||
async invoke(elementId) {
|
||||
return ActionResult.parse(await call('invoke', { ...snapshotParams(), elementId }))
|
||||
},
|
||||
async hover(target, options) {
|
||||
const relative = options?.relative === undefined ? {} : { relative: options.relative }
|
||||
if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...snapshotParams(), elementId: target, ...relative }))
|
||||
const [x, y] = target
|
||||
return HoverResult.parse(await call('hover', { ...snapshotParams(), windowId: windowId(), x, y, ...relative }))
|
||||
},
|
||||
async click(target, options) {
|
||||
const mouseButton = options?.mouseButton ?? 'left'
|
||||
const clickCount = options?.clickCount ?? 1
|
||||
if (mouseButton !== 'left' && clickCount !== 1) throw new Error('Only left double-click is supported')
|
||||
const modifiers = options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] }
|
||||
const hold = options?.holdMs === undefined ? {} : { holdMs: options.holdMs }
|
||||
const relative = options?.relative === undefined ? {} : { relative: options.relative }
|
||||
if (typeof target === 'string') {
|
||||
return action('click', { ...snapshotParams(), ...modifiers, ...hold, ...relative, elementId: target,
|
||||
...mouseButton === 'left' ? {} : { mouseButton }, ...clickCount === 1 ? {} : { clickCount },
|
||||
function createSurface(appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface {
|
||||
return {
|
||||
async show() {
|
||||
return ShowResult.parse(await call('show', { appHandle,
|
||||
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
|
||||
}))
|
||||
},
|
||||
async getAccessibilityTree(options) {
|
||||
observed.delete(appHandle)
|
||||
const state = StateResult.parse(await call('getState', { appHandle,
|
||||
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
|
||||
...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus },
|
||||
...options?.details === undefined ? {} : { details: options.details },
|
||||
}))
|
||||
if (state.appHandle !== appHandle || state.pid !== pid || state.windowId !== (selectedWindow ?? null)) {
|
||||
throw new Error('Native observation belongs to another application or window')
|
||||
}
|
||||
if (state.windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application')
|
||||
observed.set(appHandle, state)
|
||||
if (options?.emit !== false) await sink?.text(state.tree)
|
||||
return state.tree
|
||||
},
|
||||
async listActions(elementId) {
|
||||
return ActionNamesResult.parse(await call('listActions', { ...elementParams(appHandle), elementId }))
|
||||
},
|
||||
async performAction(elementId, name, argument) {
|
||||
return action('performAction', { ...elementParams(appHandle), elementId, action: name,
|
||||
...argument === undefined ? {} : { argument },
|
||||
})
|
||||
}
|
||||
const [x, y] = target
|
||||
return action('click', { ...snapshotParams(), windowId: windowId(), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative })
|
||||
},
|
||||
async setValue(elementId, value) { return action('setValue', { ...snapshotParams(), elementId, value }) },
|
||||
async selectText(elementId, text, options) {
|
||||
return action('selectText', { ...snapshotParams(), elementId, text,
|
||||
...options?.prefix === undefined ? {} : { prefix: options.prefix },
|
||||
...options?.suffix === undefined ? {} : { suffix: options.suffix },
|
||||
...options?.selectionType === undefined ? {} : { selectionType: options.selectionType },
|
||||
})
|
||||
},
|
||||
async scroll(options) {
|
||||
if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero')
|
||||
return action('scroll', {
|
||||
...snapshotParams(), windowId: windowId(),
|
||||
x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY,
|
||||
...options.relative === undefined ? {} : { relative: options.relative },
|
||||
})
|
||||
},
|
||||
async drag(from, to, options) {
|
||||
return action('drag', { ...snapshotParams(), windowId: windowId(), fromX: from[0], fromY: from[1], toX: to[0], toY: to[1],
|
||||
...options?.relative === undefined ? {} : { relative: options.relative },
|
||||
...options?.mouseButton === undefined ? {} : { mouseButton: options.mouseButton },
|
||||
...options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] },
|
||||
})
|
||||
},
|
||||
},
|
||||
async invoke(elementId) {
|
||||
return ActionResult.parse(await call('invoke', { ...elementParams(appHandle), elementId }))
|
||||
},
|
||||
async setValue(elementId, value) { return action('setValue', { ...elementParams(appHandle), elementId, value }) },
|
||||
async selectText(elementId, text, options) {
|
||||
return action('selectText', { ...elementParams(appHandle), elementId, text,
|
||||
...options?.prefix === undefined ? {} : { prefix: options.prefix },
|
||||
...options?.suffix === undefined ? {} : { suffix: options.suffix },
|
||||
...options?.selectionType === undefined ? {} : { selectionType: options.selectionType },
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
return result
|
||||
function createWindow(appHandle: CuaAppHandle, pid: number, windowId: number): CuaWindow {
|
||||
return {
|
||||
...createSurface(appHandle, pid, windowId), windowId,
|
||||
async getScreenshot(options) {
|
||||
const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId }))
|
||||
if (image.windowId !== windowId) throw new Error('Native screenshot belongs to another window')
|
||||
const screenshot: CuaScreenshot = {
|
||||
bytes: Uint8Array.from(Buffer.from(image.data, 'base64')),
|
||||
mimeType: image.mimeType,
|
||||
pixelWidth: image.pixelWidth,
|
||||
pixelHeight: image.pixelHeight,
|
||||
windowId: image.windowId,
|
||||
}
|
||||
if (options?.emit !== false) await sink?.image(screenshot)
|
||||
return screenshot
|
||||
},
|
||||
async hover(target, options) {
|
||||
const relative = options?.relative === undefined ? {} : { relative: options.relative }
|
||||
if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...elementParams(appHandle), elementId: target, ...relative }))
|
||||
const [x, y] = target
|
||||
return HoverResult.parse(await call('hover', { ...coordinateParams(appHandle, windowId), x, y, ...relative }))
|
||||
},
|
||||
async click(target, options) {
|
||||
const mouseButton = options?.mouseButton ?? 'left'
|
||||
const clickCount = options?.clickCount ?? 1
|
||||
if (mouseButton !== 'left' && clickCount !== 1) throw new Error('Only left double-click is supported')
|
||||
const modifiers = options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] }
|
||||
const hold = options?.holdMs === undefined ? {} : { holdMs: options.holdMs }
|
||||
const relative = options?.relative === undefined ? {} : { relative: options.relative }
|
||||
if (typeof target === 'string') {
|
||||
return action('click', { ...elementParams(appHandle), ...modifiers, ...hold, ...relative, elementId: target,
|
||||
...mouseButton === 'left' ? {} : { mouseButton }, ...clickCount === 1 ? {} : { clickCount },
|
||||
})
|
||||
}
|
||||
const [x, y] = target
|
||||
return action('click', { ...coordinateParams(appHandle, windowId), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative })
|
||||
},
|
||||
async scroll(options) {
|
||||
if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero')
|
||||
return action('scroll', {
|
||||
...coordinateParams(appHandle, windowId),
|
||||
x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY,
|
||||
...options.relative === undefined ? {} : { relative: options.relative },
|
||||
})
|
||||
},
|
||||
async drag(from, to, options) {
|
||||
return action('drag', { ...coordinateParams(appHandle, windowId),
|
||||
fromX: from[0], fromY: from[1], toX: to[0], toY: to[1],
|
||||
...options?.relative === undefined ? {} : { relative: options.relative },
|
||||
...options?.mouseButton === undefined ? {} : { mouseButton: options.mouseButton },
|
||||
...options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] },
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
return { createSurface, createWindow }
|
||||
}
|
||||
@@ -12,7 +12,7 @@ export type CuaToolBinding = (operation: CuaOperation) => Promise<JsonValue>
|
||||
* Connect the object API without owning an evaluator or its lifetime.
|
||||
* @param getTool - Resolves the current execution's tools.cua binding on every operation; do not capture a completed execution's binding.
|
||||
* @param onText - Optional text output captured by the caller's execution environment.
|
||||
* @returns App objects retaining tool approval, logging, image delivery and screenshot metadata. Rejections propagate without retries.
|
||||
* @returns Application and window objects using the current tool binding. Rejections propagate without retries.
|
||||
*/
|
||||
export function connectCua(getTool: () => CuaToolBinding, onText?: (text: string) => void | Promise<void>): CuaApi {
|
||||
return createCuaApi(operation => getTool()(operation), {
|
||||
|
||||
@@ -131,6 +131,8 @@ export const ScreenshotResult = NativeScreenshotResult.extend({
|
||||
/** Permissions are reported without opening system permission dialogs. */
|
||||
export const CapabilitiesResult = z.object({
|
||||
platform: z.string(), permissions: z.object({ accessibility: z.boolean(), screenRecording: z.boolean() }),
|
||||
/** Whether the provider supports application-level accessibility observations. */
|
||||
appLevelAccessibility: z.boolean(),
|
||||
sessionLocked: z.boolean().nullable(),
|
||||
methods: z.array(z.string()), coordinateSpace: z.literal('screenshot-pixels'), browser: z.literal(false),
|
||||
})
|
||||
|
||||
@@ -34,7 +34,7 @@ export interface CuaObservationSink {
|
||||
}
|
||||
/** Observations emit by default. Returned values are unaffected by this option. */
|
||||
export interface CuaObservationOptions { emit?: boolean }
|
||||
/** Full observations of the requested content; each read replaces this surface's current snapshot. */
|
||||
/** Full observations of the requested content; each read replaces the application's current snapshot. */
|
||||
export interface CuaAxOptions extends CuaObservationOptions {
|
||||
/** Reserved compatibility option; observations are never diffs. */
|
||||
disableDiffing?: boolean
|
||||
@@ -85,12 +85,8 @@ export type CuaPasteOptions = Pick<CuaParameters<'paste'>, 'format'>
|
||||
export type CuaSelectTextOptions = Pick<CuaParameters<'selectText'>, 'prefix' | 'suffix' | 'selectionType'>
|
||||
/** Positive deltas reveal content above/left; each axis is an integer from -10 to 10. */
|
||||
export type CuaScrollOptions = Pick<CuaParameters<'scroll'>, 'x' | 'y' | 'deltaLinesX' | 'deltaLinesY'> & CuaCoordinateOptions
|
||||
/** An observed AX tree and independently captured image; capture is not atomic. */
|
||||
export interface CuaObservation { state: string; screenshot: CuaScreenshot }
|
||||
/**
|
||||
* App-wide or explicitly scoped window observations and actions.
|
||||
* Capture and pointer delivery reject known AX/display frame conflicts with WINDOW_CHANGED.
|
||||
* Coordinate input also rejects resizing since the latest successful AX read or screenshot.
|
||||
* Accessibility observations and element operations shared by an application and its windows.
|
||||
* Advertised AX actions remain available; failed actions are not retried through pointer input.
|
||||
*/
|
||||
export interface CuaSurface {
|
||||
@@ -100,32 +96,21 @@ export interface CuaSurface {
|
||||
*/
|
||||
show(): Promise<CuaShowResult>
|
||||
/**
|
||||
* Read fresh AX text, with menu contents collapsed by default.
|
||||
* Read the full accessibility tree, with menu contents collapsed by default.
|
||||
* The returned tree becomes the application's current snapshot; use element ids from its latest successful read.
|
||||
* @param options - emission, menu expansion and field detail options.
|
||||
* @returns AX text with element ids.
|
||||
* @returns Full accessibility tree text with element ids, subject to the provider's traversal limits.
|
||||
*/
|
||||
getAXState(options?: CuaAxOptions): Promise<string>
|
||||
/**
|
||||
* Capture the selected window.
|
||||
* @param options - emission options.
|
||||
* @returns encoded model-request image with its format, pixel dimensions and window; use mimeType when saving or declaring the image.
|
||||
*/
|
||||
getScreenshot(options?: CuaObservationOptions): Promise<CuaScreenshot>
|
||||
/**
|
||||
* Read AX then capture.
|
||||
* @param options - observation options.
|
||||
* @returns both non-atomic observations.
|
||||
*/
|
||||
getAXStateAndScreenshot(options?: CuaAxOptions): Promise<CuaObservation>
|
||||
getAccessibilityTree(options?: CuaAxOptions): Promise<string>
|
||||
/**
|
||||
* List the actions currently advertised by an element, including attribute-setting operations.
|
||||
* @param elementId - Element id from this surface's latest observation.
|
||||
* @param elementId - Element id from the application's current snapshot.
|
||||
* @returns Provider-defined action names; empty when the element advertises none.
|
||||
*/
|
||||
listActions(elementId: CuaElementId): Promise<string[]>
|
||||
/**
|
||||
* Perform an element action, including attribute-setting operations.
|
||||
* @param elementId - Element id from this surface's latest observation.
|
||||
* @param elementId - Element id from the application's current snapshot.
|
||||
* @param action - Exact action name from listActions.
|
||||
* @param argument - Direct action argument; omit or pass null for a no-argument action, or pass the value itself for attribute setting.
|
||||
* @returns Whether the provider successfully executed the requested operation.
|
||||
@@ -134,10 +119,39 @@ export interface CuaSurface {
|
||||
/**
|
||||
* Invoke the element's default action.
|
||||
* If the element's type does not support a default action, the result's dispatched field is false.
|
||||
* @param elementId - Element id from this surface's latest observation.
|
||||
* @param elementId - Element id from the application's current snapshot.
|
||||
* @returns Whether the provider successfully executed the requested operation.
|
||||
*/
|
||||
invoke(elementId: CuaElementId): Promise<CuaActionResult>
|
||||
/**
|
||||
* Set an observed control value.
|
||||
* @param elementId - Element id from the application's current snapshot.
|
||||
* @param value - Replacement value as JSON; supported values are defined by the platform provider.
|
||||
* @returns dispatch evidence.
|
||||
*/
|
||||
setValue(elementId: CuaElementId, value: JsonValue): Promise<CuaActionResult>
|
||||
/**
|
||||
* Select one literal text occurrence through writable AXSelectedTextRange and verify its range.
|
||||
* @param elementId - Element id from the application's current snapshot.
|
||||
* @param text - Nonempty literal text; multiple matches require adjacent context.
|
||||
* @param options - Adjacent prefix/suffix and selection or cursor placement.
|
||||
* @returns Dispatch evidence; rejects unsupported, ambiguous or changed text without keyboard fallback.
|
||||
*/
|
||||
selectText(elementId: CuaElementId, text: string, options?: CuaSelectTextOptions): Promise<CuaActionResult>
|
||||
}
|
||||
/**
|
||||
* An explicit window with screenshot and pointer input operations.
|
||||
* Capture and pointer delivery reject known AX/display frame conflicts with WINDOW_CHANGED.
|
||||
* Coordinate input also rejects resizing since the latest successful AX read or screenshot.
|
||||
*/
|
||||
export interface CuaWindow extends CuaSurface {
|
||||
readonly windowId: number
|
||||
/**
|
||||
* Capture the selected window.
|
||||
* @param options - emission options.
|
||||
* @returns encoded model-request image with its format, pixel dimensions and window; use mimeType when saving or declaring the image.
|
||||
*/
|
||||
getScreenshot(options?: CuaObservationOptions): Promise<CuaScreenshot>
|
||||
/**
|
||||
* Reveal advertised AX alternate UI, otherwise move the system cursor without pressing a button.
|
||||
* The system-pointer route requires the app to be foreground and the point unobscured; never activates or raises it.
|
||||
@@ -156,21 +170,6 @@ export interface CuaSurface {
|
||||
* @returns dispatch evidence.
|
||||
*/
|
||||
click(target: CuaElementId | readonly [number, number], options?: CuaClickOptions): Promise<CuaActionResult>
|
||||
/**
|
||||
* Set an observed control value.
|
||||
* @param elementId - element id from this surface's latest observation.
|
||||
* @param value - Replacement value as JSON; supported values are defined by the platform provider.
|
||||
* @returns dispatch evidence.
|
||||
*/
|
||||
setValue(elementId: CuaElementId, value: JsonValue): Promise<CuaActionResult>
|
||||
/**
|
||||
* Select one literal text occurrence through writable AXSelectedTextRange and verify its range.
|
||||
* @param elementId - Text control element id from this surface's latest observation.
|
||||
* @param text - Nonempty literal text; multiple matches require adjacent context.
|
||||
* @param options - Adjacent prefix/suffix and selection or cursor placement.
|
||||
* @returns Dispatch evidence; rejects unsupported, ambiguous or changed text without keyboard fallback.
|
||||
*/
|
||||
selectText(elementId: CuaElementId, text: string, options?: CuaSelectTextOptions): Promise<CuaActionResult>
|
||||
/**
|
||||
* Scroll at a screenshot-local point or integer ratio.
|
||||
* @param options - point, coordinate mode and line deltas; at least one delta must be nonzero.
|
||||
@@ -186,14 +185,7 @@ export interface CuaSurface {
|
||||
*/
|
||||
drag(from: readonly [number, number], to: readonly [number, number], options?: CuaDragOptions): Promise<CuaActionResult>
|
||||
}
|
||||
/** One explicit window; keyboard operations remain app-scoped because they follow focus. */
|
||||
export interface CuaWindow extends CuaSurface { readonly windowId: number }
|
||||
/**
|
||||
* Native app handle supporting direct observations and actions without window enumeration.
|
||||
* Screenshots and coordinates reuse the latest AX observation's focused on-screen,
|
||||
* nonzero-size window, or require exactly one such window. Read fresh AX to follow
|
||||
* focus changes; explicit window objects retain their selected target.
|
||||
*/
|
||||
/** Application-level accessibility, window discovery, keyboard input and clipboard operations. */
|
||||
export interface CuaApp extends CuaSurface {
|
||||
readonly pid: number
|
||||
readonly appHandle: CuaAppHandle
|
||||
@@ -211,7 +203,7 @@ export interface CuaApp extends CuaSurface {
|
||||
/**
|
||||
* Observe a specific window.
|
||||
* @param windowId - ID obtained from getWindows.
|
||||
* @returns a reusable window object with its initial AX state retained silently; getAXState emits it.
|
||||
* @returns A window object with its initial accessibility tree read silently; this updates the application's current snapshot.
|
||||
*/
|
||||
getWindow(windowId: number): Promise<CuaWindow>
|
||||
/**
|
||||
@@ -265,7 +257,7 @@ export interface CuaApi {
|
||||
/**
|
||||
* Select an app, obtain consent, launch it in the background if necessary, and silently retain initial AX state.
|
||||
* @param selector - Exact name, bundle ID, absolute application path or running PID string; ambiguous names reject.
|
||||
* @returns a handle tied to this process lifetime.
|
||||
* @returns An application handle tied to this process lifetime; its initial tree read becomes the current snapshot.
|
||||
*/
|
||||
getApp(selector: string): Promise<CuaApp>
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
/** Window facade using the same observed surface operations as the application facade. */
|
||||
import { z } from 'zod'
|
||||
import { createCuaSurface, keyChord } from './api.ts'
|
||||
import { createCuaObjects, keyChord } from './api.ts'
|
||||
import { ActionResult, CapabilitiesResult } from './protocol.ts'
|
||||
import { LaunchEntriesResult, LaunchResult, SelectedWindowsResult, WindowDiscoveryResult, WindowRequest } from './window-protocol.ts'
|
||||
import type { CuaAppHandle, CuaObservationOptions, CuaObservationSink } from './types.ts'
|
||||
@@ -43,23 +43,23 @@ export function createCuaWindowApi(dispatch: CuaWindowDispatch, sink?: CuaObserv
|
||||
const { windowId, pid } = selectedWindow
|
||||
const windowHandle = selectedWindow.windowHandle as CuaWindowHandle
|
||||
const appHandle = selectedWindow.windowHandle as CuaAppHandle
|
||||
const surface = createCuaSurface(async (operation) => {
|
||||
const params = z.record(z.string(), z.json()).parse(operation.params)
|
||||
delete params.appHandle
|
||||
delete params.windowId
|
||||
const result = await call(operation.method, { ...params, windowHandle })
|
||||
if (operation.method !== 'getState') return result
|
||||
const state = z.record(z.string(), z.json()).parse(result)
|
||||
return { ...state, appHandle }
|
||||
}, sink, appHandle, pid, windowId)
|
||||
const window: CuaDesktopWindow = {
|
||||
...surface, windowId, windowHandle,
|
||||
...createCuaObjects(async (operation) => {
|
||||
const params = z.record(z.string(), z.json()).parse(operation.params)
|
||||
delete params.appHandle
|
||||
delete params.windowId
|
||||
const result = await call(operation.method, { ...params, windowHandle })
|
||||
if (operation.method !== 'getState') return result
|
||||
const state = z.record(z.string(), z.json()).parse(result)
|
||||
return { ...state, appHandle }
|
||||
}, sink).createWindow(appHandle, pid, windowId),
|
||||
windowHandle,
|
||||
async activate() { return ActionResult.parse(await call('activate', { windowHandle })) },
|
||||
async typeText(text) { return ActionResult.parse(await call('typeText', { windowHandle, text })) },
|
||||
async paste(text, options) { return ActionResult.parse(await call('paste', { windowHandle, text, ...options })) },
|
||||
async pressKey(key) { return ActionResult.parse(await call('pressKey', { windowHandle, ...keyChord(key) })) },
|
||||
}
|
||||
await window.getAXState({ emit: false })
|
||||
await window.getAccessibilityTree({ emit: false })
|
||||
windows.push(window)
|
||||
}
|
||||
return windows
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import type { Branded } from '@deepseek-ai/dsh-brand'
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import type { z } from 'zod'
|
||||
import type { CuaActionResult, CuaCapabilities, CuaObservationOptions, CuaPasteOptions, CuaSurface, CuaWindowInfo } from './types.ts'
|
||||
import type { CuaActionResult, CuaCapabilities, CuaObservationOptions, CuaPasteOptions, CuaWindow, CuaWindowInfo } from './types.ts'
|
||||
import type { LaunchEntriesResult, WindowRequest } from './window-protocol.ts'
|
||||
|
||||
/** Connection-owned grant for exactly one observed window lifetime. */
|
||||
@@ -14,7 +14,7 @@ export type CuaWindowDispatch = (operation: CuaWindowOperation) => Promise<JsonV
|
||||
/** Discovered launch entry; its identifier is accepted by launch, not by window selection. */
|
||||
export type CuaLaunchEntry = z.output<typeof LaunchEntriesResult>[number]
|
||||
/** Selected window; keyboard and clipboard operations require this window in front. */
|
||||
export interface CuaDesktopWindow extends CuaSurface {
|
||||
export interface CuaDesktopWindow extends CuaWindow {
|
||||
windowId: number
|
||||
windowHandle: CuaWindowHandle
|
||||
/** Bring this window forward. @returns Dispatch evidence, not application success. */
|
||||
@@ -32,7 +32,7 @@ export interface CuaWindowApi {
|
||||
getCapabilities(options?: CuaObservationOptions): Promise<CuaCapabilities>
|
||||
/** Discover top-level windows without reading AX. @param options Output options. @returns Window metadata. */
|
||||
listWindows(options?: CuaObservationOptions): Promise<CuaWindowInfo[]>
|
||||
/** Authorize selected windows and read their initial AX silently. A failure may leave earlier grants in place.
|
||||
/** Authorize selected windows and read their initial accessibility trees silently. A failure may leave earlier grants in place.
|
||||
* @param ids IDs from listWindows; duplicates reject. @returns Windows in requested order.
|
||||
*/
|
||||
getWindows(ids: readonly number[]): Promise<CuaDesktopWindow[]>
|
||||
|
||||
@@ -4,7 +4,7 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import { brandString } from '@deepseek-ai/dsh-brand'
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import { createCuaApi } from '../src/api.ts'
|
||||
import type { CuaApp, CuaElementId, CuaOperation, CuaScreenshot } from '../src/types.ts'
|
||||
import type { CuaApp, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts'
|
||||
|
||||
/** Element ids are opaque strings; these fixtures use the macOS decimal form. */
|
||||
const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
|
||||
@@ -13,7 +13,7 @@ const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG
|
||||
const bounds = { x: -640, y: 100, width: 640, height: 480 }
|
||||
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor' }
|
||||
const delivered = { dispatched: true }
|
||||
const capabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
|
||||
const capabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
|
||||
|
||||
function windowInfo(windowId: number) {
|
||||
return { windowId, pid: 17, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null }
|
||||
@@ -77,9 +77,10 @@ describe('application discovery', () => {
|
||||
const app = await f.api.getApp('Editor')
|
||||
expect(app.pid).toBe(17)
|
||||
expect(app.appHandle).toBe('app-17')
|
||||
expect(await app.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
|
||||
const window = await app.getWindow(23)
|
||||
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
|
||||
expect(f.dispatch.mock.calls.map(([operation]) => operation.method)).toEqual([
|
||||
'getCapabilities', 'listApps', 'getApp', 'getState', 'screenshot',
|
||||
'getCapabilities', 'listApps', 'getApp', 'getState', 'getState', 'screenshot',
|
||||
])
|
||||
expect(await app.getWindows()).toEqual([windowInfo(23)])
|
||||
expect(f.sink.text).not.toHaveBeenCalled()
|
||||
@@ -154,62 +155,63 @@ describe('hover across observations', () => {
|
||||
it('moves to an AX element then into a popup without inserting clicks or restoring a point', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
f.dispatch.mockClear()
|
||||
expect(await app.hover(element('0'))).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
|
||||
await app.getAXStateAndScreenshot({ emit: false })
|
||||
expect(await app.hover([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
|
||||
expect(await window.hover(element('0'))).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
|
||||
await window.getAccessibilityTree({ emit: false })
|
||||
await window.getScreenshot({ emit: false })
|
||||
expect(await window.hover([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
|
||||
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } },
|
||||
{ method: 'getState', params: { appHandle: 'app-17' } },
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0' } },
|
||||
{ method: 'getState', params: { appHandle: 'app-17', windowId: 23 } },
|
||||
{ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } },
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 30, y: 40 } },
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 23, x: 30, y: 40 } },
|
||||
])
|
||||
const window = await app.getWindow(24)
|
||||
await window.hover([5, 6])
|
||||
const other = await app.getWindow(24)
|
||||
await other.hover([5, 6])
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'hover', params: {
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 24, x: 5, y: 6,
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-4', windowId: 24, x: 5, y: 6,
|
||||
} })
|
||||
f.dispatch.mockRejectedValueOnce(new Error('WINDOW_CHANGED'))
|
||||
await expect(window.hover([5, 6])).rejects.toThrow('WINDOW_CHANGED')
|
||||
await expect(other.hover([5, 6])).rejects.toThrow('WINDOW_CHANGED')
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(7)
|
||||
})
|
||||
})
|
||||
|
||||
describe('pointer buttons and modifiers', () => {
|
||||
it.each([0, 800])('preserves %d ms for app and window clicks at elements and coordinates', async (holdMs) => {
|
||||
it.each([0, 800])('preserves %d ms for window clicks at elements and coordinates', async (holdMs) => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
for (const surface of [app, window]) {
|
||||
for (const target of [element('0'), [10, 20] as const]) {
|
||||
await surface.click(target, { holdMs, mouseButton: 'right' })
|
||||
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ method: 'click', params: { holdMs, mouseButton: 'right' } })
|
||||
f.dispatch.mockClear()
|
||||
await expect(surface.click(target, { holdMs: -1 })).rejects.toThrow('holdMs')
|
||||
expect(f.dispatch).not.toHaveBeenCalled()
|
||||
}
|
||||
for (const target of [element('0'), [10, 20] as const]) {
|
||||
await window.click(target, { holdMs, mouseButton: 'right' })
|
||||
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ method: 'click', params: { holdMs, mouseButton: 'right' } })
|
||||
f.dispatch.mockClear()
|
||||
await expect(window.click(target, { holdMs: -1 })).rejects.toThrow('holdMs')
|
||||
expect(f.dispatch).not.toHaveBeenCalled()
|
||||
}
|
||||
})
|
||||
|
||||
it('retains middle-button and modifier intent through click and drag dispatch', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
await app.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] })
|
||||
const window = await app.getWindow(23)
|
||||
await window.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', mouseButton: 'middle', modifiers: ['shift'],
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', mouseButton: 'middle', modifiers: ['shift'],
|
||||
} })
|
||||
await app.click([10, 20], { modifiers: ['command'] })
|
||||
await window.click([10, 20], { modifiers: ['command'] })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20,
|
||||
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20,
|
||||
mouseButton: 'left', clickCount: 1, modifiers: ['command'],
|
||||
} })
|
||||
await app.drag([10, 20], [30, 40], { mouseButton: 'middle', modifiers: ['shift', 'option'] })
|
||||
await window.drag([10, 20], [30, 40], { mouseButton: 'middle', modifiers: ['shift', 'option'] })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'drag', params: {
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, fromX: 10, fromY: 20, toX: 30, toY: 40,
|
||||
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', fromX: 10, fromY: 20, toX: 30, toY: 40,
|
||||
mouseButton: 'middle', modifiers: ['shift', 'option'],
|
||||
} })
|
||||
const count = f.dispatch.mock.calls.length
|
||||
await expect(app.click(element('0'), { mouseButton: 'middle', clickCount: 2 })).rejects.toThrow('Only left double-click')
|
||||
await expect(window.click(element('0'), { mouseButton: 'middle', clickCount: 2 })).rejects.toThrow('Only left double-click')
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(count)
|
||||
})
|
||||
})
|
||||
@@ -307,30 +309,87 @@ describe('element default invocation', () => {
|
||||
})
|
||||
|
||||
describe('application and window observations', () => {
|
||||
it('keeps the latest element id scope separate for an app and each selected window', async () => {
|
||||
it('shares the latest tree read across an application and each selected window', async () => {
|
||||
const f = fixture()
|
||||
f.setWindows([23, 24])
|
||||
const app = await f.api.getApp('Editor')
|
||||
expect(await app.getWindows({ emit: false })).toEqual([windowInfo(23), windowInfo(24)])
|
||||
const first = await app.getWindow(23)
|
||||
const second = await app.getWindow(24)
|
||||
await app.click(element('0'))
|
||||
await first.setValue(element('1'), 'First document')
|
||||
await second.click(element('2'))
|
||||
await app.setValue(element('1'), 'First document')
|
||||
await first.click(element('2'))
|
||||
await second.click(element('3'))
|
||||
expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } },
|
||||
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '1', value: 'First document' } },
|
||||
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '1', value: 'First document' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3' } },
|
||||
])
|
||||
expect(first.windowId).toBe(23)
|
||||
expect(second.windowId).toBe(24)
|
||||
expect(f.sink.text.mock.calls.map(([text]) => text)).toEqual([])
|
||||
await expect(second.getAXState({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4')
|
||||
await expect(second.getAccessibilityTree({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4')
|
||||
await second.click(element('3'))
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3' } })
|
||||
await app.getAccessibilityTree({ emit: false })
|
||||
await first.invoke(element('4'))
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4' } })
|
||||
expect(f.sink.text).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps separate snapshots for different application handles', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
f.dispatch.mockResolvedValueOnce({ appHandle: 'app-18', pid: 18, name: 'Utility' })
|
||||
f.dispatch.mockResolvedValueOnce(state({ appHandle: 'app-18', pid: 18, snapshotId: 'other', windows: [] }))
|
||||
const other = await f.api.getApp('Utility')
|
||||
await app.setValue(element('0'), 'Editor')
|
||||
await other.setValue(element('0'), 'Utility')
|
||||
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
|
||||
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', value: 'Editor' } },
|
||||
{ method: 'setValue', params: { appHandle: 'app-18', snapshotId: 'other', elementId: '0', value: 'Utility' } },
|
||||
])
|
||||
})
|
||||
|
||||
it('shares one current snapshot between repeated handles for the same application', async () => {
|
||||
const f = fixture()
|
||||
const first = await f.api.getApp('Editor')
|
||||
const second = await f.api.getApp('Editor')
|
||||
expect(second.appHandle).toBe(first.appHandle)
|
||||
await first.setValue(element('1'), 'Shared')
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'setValue', params: {
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '1', value: 'Shared',
|
||||
} })
|
||||
})
|
||||
|
||||
it('exposes screenshots and pointer input only on explicit windows', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
expect(app).toHaveProperty('getAccessibilityTree')
|
||||
for (const method of ['getScreenshot', 'click', 'hover', 'scroll', 'drag']) expect(app).not.toHaveProperty(method)
|
||||
const window = await app.getWindow(23)
|
||||
for (const method of ['getScreenshot', 'click', 'hover', 'scroll', 'drag']) expect(window).toHaveProperty(method)
|
||||
})
|
||||
|
||||
it('attaches the shared snapshot to coordinates only within its window scope', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const first = await app.getWindow(23)
|
||||
await first.click([10, 20])
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
|
||||
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
|
||||
} })
|
||||
const second = await app.getWindow(24)
|
||||
await first.click([10, 20])
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
|
||||
appHandle: 'app-17', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
|
||||
} })
|
||||
await second.click([10, 20])
|
||||
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-3', windowId: 24 } })
|
||||
await app.getAccessibilityTree({ emit: false })
|
||||
await first.click([10, 20])
|
||||
expect(f.dispatch.mock.calls.at(-1)?.[0]).toMatchObject({ params: { snapshotId: 'snapshot-4', windowId: 23 } })
|
||||
})
|
||||
|
||||
it('keeps a validated AXless window usable for screenshots and coordinate input', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
@@ -357,17 +416,7 @@ describe('application and window observations', () => {
|
||||
expect(f.dispatch.mock.calls.slice(calls).map(([operation]) => operation.method)).toEqual(['getState'])
|
||||
})
|
||||
|
||||
it.each([{ ids: [] }, { ids: [23, 24] }])('requires an explicit window when discovery yields $ids', async ({ ids }) => {
|
||||
const f = fixture()
|
||||
f.setWindows(ids)
|
||||
const app = await f.api.getApp('Editor')
|
||||
const calls = f.dispatch.mock.calls.length
|
||||
await expect(app.click([1, 2])).rejects.toThrow('Select a window')
|
||||
await expect(app.getScreenshot()).rejects.toThrow('Select a window')
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(calls)
|
||||
})
|
||||
|
||||
it('selects the one visible nonzero window while preserving menu and hidden windows in discovery', async () => {
|
||||
it('captures and clicks the explicitly selected window among several', async () => {
|
||||
const f = fixture()
|
||||
const windows = [
|
||||
windowInfo(23),
|
||||
@@ -378,46 +427,28 @@ describe('application and window observations', () => {
|
||||
f.setWindowRecords(windows)
|
||||
const app = await f.api.getApp('Editor')
|
||||
expect(await app.getWindows({ emit: false })).toEqual(windows)
|
||||
await app.getScreenshot()
|
||||
const window = await app.getWindow(23)
|
||||
await window.getScreenshot()
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } })
|
||||
await app.click([10, 20])
|
||||
await window.click([10, 20])
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
|
||||
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
|
||||
} })
|
||||
const hidden = await app.getWindow(25)
|
||||
await hidden.click([10, 20])
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 25, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
|
||||
appHandle: 'app-17', windowId: 25, snapshotId: 'snapshot-3', x: 10, y: 20, mouseButton: 'left', clickCount: 1,
|
||||
} })
|
||||
})
|
||||
|
||||
it('requires explicit selection when every window is hidden or has zero size', async () => {
|
||||
it('captures a window without reading its accessibility tree', async () => {
|
||||
const f = fixture()
|
||||
f.setWindowRecords([
|
||||
{ ...windowInfo(24), bounds: { ...bounds, width: 0, height: 0 } },
|
||||
{ ...windowInfo(25), isOnScreen: false },
|
||||
])
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
const calls = f.dispatch.mock.calls.length
|
||||
await expect(app.getScreenshot()).rejects.toThrow('Select a window')
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(calls)
|
||||
})
|
||||
|
||||
it('captures the focused application window when several visible windows exist', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
f.dispatch.mockResolvedValueOnce(state({ focusedWindowId: 24, windows: [windowInfo(23), windowInfo(24)] }))
|
||||
await app.getAXState()
|
||||
await app.getScreenshot()
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 24 } })
|
||||
})
|
||||
|
||||
it('does not select a focused window absent from the application observation', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
f.dispatch.mockResolvedValueOnce(state({ focusedWindowId: 99, windows: [windowInfo(23), windowInfo(24)] }))
|
||||
await app.getAXState()
|
||||
await expect(app.getScreenshot()).rejects.toThrow('Select a window')
|
||||
await window.getScreenshot()
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(calls + 1)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } })
|
||||
})
|
||||
|
||||
const identityMismatches: Record<string, JsonValue>[] = [
|
||||
@@ -427,9 +458,9 @@ describe('application and window observations', () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
f.dispatch.mockResolvedValueOnce(state(identity))
|
||||
await expect(app.getAXState()).rejects.toThrow('another application or window')
|
||||
await expect(app.getAccessibilityTree()).rejects.toThrow('another application or window')
|
||||
const calls = f.dispatch.mock.calls.length
|
||||
await expect(app.click(element('0'))).rejects.toThrow('Read getAXState')
|
||||
await expect(app.setValue(element('0'), 'stale')).rejects.toThrow('Read getAccessibilityTree')
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(calls)
|
||||
})
|
||||
|
||||
@@ -438,22 +469,40 @@ describe('application and window observations', () => {
|
||||
const app = await f.api.getApp('Editor')
|
||||
const otherWindow = { ...windowInfo(23), pid: 18 }
|
||||
f.dispatch.mockResolvedValueOnce(state({ windows: [otherWindow] }))
|
||||
await expect(app.getAXState()).rejects.toThrow('windows belong to another application')
|
||||
await expect(app.getAccessibilityTree()).rejects.toThrow('windows belong to another application')
|
||||
f.dispatch.mockResolvedValueOnce([otherWindow])
|
||||
await expect(app.getWindows()).rejects.toThrow('windows belong to another application')
|
||||
expect(f.sink.text).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('clears a failed refresh and obtains new state before a later screenshot', async () => {
|
||||
it('accepts coordinate input without a current snapshot', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
f.dispatch.mockRejectedValueOnce(new Error('Native app is busy'))
|
||||
await expect(window.getAccessibilityTree()).rejects.toThrow('Native app is busy')
|
||||
await window.click([10, 20])
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
|
||||
appHandle: 'app-17', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
|
||||
} })
|
||||
})
|
||||
|
||||
it('clears a failed refresh and requires a new read before element operations', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
const failure = new Error('Native app is busy')
|
||||
f.dispatch.mockRejectedValueOnce(failure)
|
||||
await expect(app.getAXState()).rejects.toBe(failure)
|
||||
await expect(app.setValue(element('0'), 'stale')).rejects.toThrow('Read getAXState')
|
||||
await expect(app.getAccessibilityTree()).rejects.toBe(failure)
|
||||
await expect(window.setValue(element('0'), 'stale')).rejects.toThrow('Read getAccessibilityTree')
|
||||
await window.getAccessibilityTree({ emit: false })
|
||||
f.dispatch.mockRejectedValueOnce(failure)
|
||||
await expect(window.getAccessibilityTree()).rejects.toBe(failure)
|
||||
await expect(app.invoke(element('0'))).rejects.toThrow('Read getAccessibilityTree')
|
||||
f.sink.text.mockClear()
|
||||
await app.getScreenshot()
|
||||
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation.method)).toEqual(['getState', 'screenshot'])
|
||||
const calls = f.dispatch.mock.calls.length
|
||||
await window.getScreenshot()
|
||||
expect(f.dispatch.mock.calls.slice(calls).map(([operation]) => operation.method)).toEqual(['screenshot'])
|
||||
expect(f.sink.text).not.toHaveBeenCalled()
|
||||
expect(f.sink.image).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
@@ -466,13 +515,15 @@ describe('screenshots', () => {
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(24)
|
||||
f.sink.text.mockClear()
|
||||
const observation = await window.getAXStateAndScreenshot()
|
||||
expect(observation.state).toBe('Tree 3')
|
||||
expect(observation.screenshot).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 24 })
|
||||
const tree = await window.getAccessibilityTree()
|
||||
const image = await window.getScreenshot()
|
||||
expect(tree).toBe('Tree 3')
|
||||
expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 24 })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 24 } })
|
||||
expect(f.sink.text).toHaveBeenCalledWith('Tree 3')
|
||||
expect(f.sink.image).toHaveBeenCalledWith(observation.screenshot)
|
||||
await window.getAXStateAndScreenshot({ emit: false })
|
||||
expect(f.sink.image).toHaveBeenCalledWith(image)
|
||||
await window.getAccessibilityTree({ emit: false })
|
||||
await window.getScreenshot({ emit: false })
|
||||
expect(f.sink.text).toHaveBeenCalledTimes(1)
|
||||
expect(f.sink.image).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
@@ -480,17 +531,19 @@ describe('screenshots', () => {
|
||||
it('rejects a screenshot captured from another window before emitting it', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
f.dispatch.mockResolvedValueOnce(screenshot({ windowId: 24 }))
|
||||
await expect(app.getScreenshot()).rejects.toThrow('another window')
|
||||
await expect(window.getScreenshot()).rejects.toThrow('another window')
|
||||
expect(f.sink.image).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each(['image/jpeg', 'image/webp', 'image/gif'])('preserves Host-admitted %s bytes and metadata in the return value and image sink', async (mimeType) => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
const data = Buffer.from('admitted request image').toString('base64')
|
||||
f.dispatch.mockResolvedValueOnce(screenshot({ mimeType, data, pixelWidth: 320, pixelHeight: 240 }))
|
||||
const image = await app.getScreenshot()
|
||||
const image = await window.getScreenshot()
|
||||
expect(image).toEqual({ bytes: Uint8Array.from(Buffer.from(data, 'base64')), mimeType, pixelWidth: 320, pixelHeight: 240, windowId: 23 })
|
||||
expect(f.sink.image).toHaveBeenCalledExactlyOnceWith(image)
|
||||
})
|
||||
@@ -501,21 +554,22 @@ describe('input actions', () => {
|
||||
it('passes explicit relative mode to every coordinate gesture and rejects it on element ids', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
await app.click([500, 250], { relative: true })
|
||||
await app.hover([500, 250], { relative: true })
|
||||
await app.drag([0, 0], [999, 999], { relative: true })
|
||||
await app.scroll({ x: 500, y: 250, relative: true, deltaLinesX: 0, deltaLinesY: -1 })
|
||||
const window = await app.getWindow(23)
|
||||
await window.click([500, 250], { relative: true })
|
||||
await window.hover([500, 250], { relative: true })
|
||||
await window.drag([0, 0], [999, 999], { relative: true })
|
||||
await window.scroll({ x: 500, y: 250, relative: true, deltaLinesX: 0, deltaLinesY: -1 })
|
||||
expect(f.dispatch.mock.calls.slice(-4).map(([request]) => request.params)).toEqual([
|
||||
expect.objectContaining({ x: 500, y: 250, relative: true }),
|
||||
expect.objectContaining({ x: 500, y: 250, relative: true }),
|
||||
expect.objectContaining({ fromX: 0, toX: 999, relative: true }),
|
||||
expect.objectContaining({ x: 500, y: 250, relative: true }),
|
||||
])
|
||||
await expect(app.click(element('1'), { relative: true })).rejects.toThrow()
|
||||
await expect(app.hover(element('1'), { relative: true })).rejects.toThrow()
|
||||
await expect(window.click(element('1'), { relative: true })).rejects.toThrow()
|
||||
await expect(window.hover(element('1'), { relative: true })).rejects.toThrow()
|
||||
})
|
||||
|
||||
it('keeps the selected app and window when scroll options include another target identity', async () => {
|
||||
it('keeps the selected window when scroll options include another target identity', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
@@ -534,57 +588,61 @@ describe('input actions', () => {
|
||||
it('dispatches window-point gestures and reports delivery without claiming the action succeeded', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
expect(await app.click([10, 20], { mouseButton: 'right' })).toEqual(delivered)
|
||||
expect(await app.click([11, 21], { clickCount: 2 })).toEqual(delivered)
|
||||
expect(await app.scroll({ x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 })).toEqual(delivered)
|
||||
expect(await app.drag([1, 2], [30, 40])).toEqual(delivered)
|
||||
const window = await app.getWindow(23)
|
||||
expect(await window.click([10, 20], { mouseButton: 'right' })).toEqual(delivered)
|
||||
expect(await window.click([11, 21], { clickCount: 2 })).toEqual(delivered)
|
||||
expect(await window.scroll({ x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 })).toEqual(delivered)
|
||||
expect(await window.drag([1, 2], [30, 40])).toEqual(delivered)
|
||||
expect(f.dispatch.mock.calls.slice(-4).map(([operation]) => operation)).toEqual([
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'right', clickCount: 1 } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 11, y: 21, mouseButton: 'left', clickCount: 2 } },
|
||||
{ method: 'scroll', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } },
|
||||
{ method: 'drag', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, fromX: 1, fromY: 2, toX: 30, toY: 40 } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, mouseButton: 'right', clickCount: 1 } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 11, y: 21, mouseButton: 'left', clickCount: 2 } },
|
||||
{ method: 'scroll', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } },
|
||||
{ method: 'drag', params: { appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', fromX: 1, fromY: 2, toX: 30, toY: 40 } },
|
||||
])
|
||||
})
|
||||
|
||||
const invalidActions: Array<[string, (app: CuaApp) => Promise<unknown>]> = [
|
||||
['right double click', app => app.click([1, 2], { mouseButton: 'right', clickCount: 2 })],
|
||||
['right AX double click', app => app.click(element('0'), { mouseButton: 'right', clickCount: 2 })],
|
||||
['empty hover element id', app => app.hover(element(''))],
|
||||
['negative hover point', app => app.hover([-1, 0])],
|
||||
['nonfinite hover point', app => app.hover([NaN, 0])],
|
||||
['empty click element id', app => app.click(element(''))],
|
||||
['nonfinite coordinate', app => app.click([NaN, 0])],
|
||||
['negative coordinate', app => app.click([-1, 0])],
|
||||
['empty value element id', app => app.setValue(element(''), 'value')],
|
||||
['zero scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0, deltaLinesY: 0 })],
|
||||
['oversized scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })],
|
||||
['fractional scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })],
|
||||
['negative drag point', app => app.drag([0, 0], [-1, 3])],
|
||||
['empty chord', app => app.pressKey('')],
|
||||
['missing key', app => app.pressKey('super+')],
|
||||
['empty plus modifier', app => app.pressKey('++')],
|
||||
['extra plus delimiter', app => app.pressKey('cmd+++')],
|
||||
['unknown modifier', app => app.pressKey('unknown+a')],
|
||||
type Surfaces = { app: CuaApp; window: CuaWindow }
|
||||
const invalidActions: Array<[string, (surfaces: Surfaces) => Promise<unknown>]> = [
|
||||
['right double click', ({ window }) => window.click([1, 2], { mouseButton: 'right', clickCount: 2 })],
|
||||
['right AX double click', ({ window }) => window.click(element('0'), { mouseButton: 'right', clickCount: 2 })],
|
||||
['empty hover element id', ({ window }) => window.hover(element(''))],
|
||||
['negative hover point', ({ window }) => window.hover([-1, 0])],
|
||||
['nonfinite hover point', ({ window }) => window.hover([NaN, 0])],
|
||||
['empty click element id', ({ window }) => window.click(element(''))],
|
||||
['nonfinite coordinate', ({ window }) => window.click([NaN, 0])],
|
||||
['negative coordinate', ({ window }) => window.click([-1, 0])],
|
||||
['empty value element id', ({ window }) => window.setValue(element(''), 'value')],
|
||||
['zero scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0, deltaLinesY: 0 })],
|
||||
['oversized scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })],
|
||||
['fractional scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })],
|
||||
['negative drag point', ({ window }) => window.drag([0, 0], [-1, 3])],
|
||||
['empty chord', ({ app }) => app.pressKey('')],
|
||||
['missing key', ({ app }) => app.pressKey('super+')],
|
||||
['empty plus modifier', ({ app }) => app.pressKey('++')],
|
||||
['extra plus delimiter', ({ app }) => app.pressKey('cmd+++')],
|
||||
['unknown modifier', ({ app }) => app.pressKey('unknown+a')],
|
||||
]
|
||||
it.each<[string, (app: CuaApp) => Promise<unknown>]>([
|
||||
['getState windowId: Too small: expected number to be >0', app => app.getWindow(0)],
|
||||
['selectText text: Too small: expected string to have >=1 characters', app => app.selectText(element('0'), '')],
|
||||
['hover x: Invalid input: expected number, received NaN', app => app.hover([NaN, 0])],
|
||||
['hover x: Too big: expected number to be <=999', app => app.hover([1000, 0], { relative: true })],
|
||||
['click relative: not accepted for these arguments', app => app.click(element('650'), { relative: true })],
|
||||
it.each<[string, (surfaces: Surfaces) => Promise<unknown>]>([
|
||||
['getState windowId: Too small: expected number to be >0', ({ app }) => app.getWindow(0)],
|
||||
['selectText text: Too small: expected string to have >=1 characters', ({ app }) => app.selectText(element('0'), '')],
|
||||
['hover x: Invalid input: expected number, received NaN', ({ window }) => window.hover([NaN, 0])],
|
||||
['hover x: Too big: expected number to be <=999', ({ window }) => window.hover([1000, 0], { relative: true })],
|
||||
['click relative: not accepted for these arguments', ({ window }) => window.click(element('650'), { relative: true })],
|
||||
])('reports a readable validation failure: %s', async (message, run) => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
f.dispatch.mockClear()
|
||||
await expect(run(app)).rejects.toMatchObject({ name: 'Error', message })
|
||||
await expect(run({ app, window })).rejects.toMatchObject({ name: 'Error', message })
|
||||
expect(f.dispatch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each(invalidActions)('rejects %s without invoking native input', async (_label, run) => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
const calls = f.dispatch.mock.calls.length
|
||||
await expect(run(app)).rejects.toThrow()
|
||||
await expect(run({ app, window })).rejects.toThrow()
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(calls)
|
||||
})
|
||||
|
||||
@@ -621,22 +679,24 @@ describe('input actions', () => {
|
||||
it('retains the snapshot identity for element right and double clicks', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
await app.click(element('12'), { mouseButton: 'right' })
|
||||
await app.click(element('13'), { clickCount: 2 })
|
||||
const window = await app.getWindow(23)
|
||||
await window.click(element('12'), { mouseButton: 'right' })
|
||||
await window.click(element('13'), { clickCount: 2 })
|
||||
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '12', mouseButton: 'right' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '13', clickCount: 2 } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', mouseButton: 'right' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', clickCount: 2 } },
|
||||
])
|
||||
})
|
||||
|
||||
it('dispatches an opaque provider element id unchanged', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
await app.click(element('42.12345'))
|
||||
await app.hover(element('42.12345'))
|
||||
const window = await app.getWindow(23)
|
||||
await window.click(element('42.12345'))
|
||||
await window.hover(element('42.12345'))
|
||||
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345' } },
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } },
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } },
|
||||
])
|
||||
})
|
||||
|
||||
@@ -651,13 +711,14 @@ describe('input actions', () => {
|
||||
it('propagates native failures once and rejects malformed delivery claims', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
const failure = new Error('Application was restarted; select it again')
|
||||
f.dispatch.mockRejectedValueOnce(failure)
|
||||
await expect(app.typeText('hello')).rejects.toBe(failure)
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(3)
|
||||
f.dispatch.mockResolvedValueOnce({ dispatched: 'yes' })
|
||||
await expect(app.click(element('0'))).rejects.toThrow()
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(4)
|
||||
f.dispatch.mockResolvedValueOnce({ dispatched: 'yes' })
|
||||
await expect(window.click(element('0'))).rejects.toThrow()
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(5)
|
||||
})
|
||||
|
||||
it('rejects invalid selection inputs before dispatch', async () => {
|
||||
@@ -674,8 +735,8 @@ describe('input actions', () => {
|
||||
it('expands menu contents and details only for the requested observation', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
await app.getAXState({ includeMenus: true, details: true, emit: false })
|
||||
await app.getAccessibilityTree({ includeMenus: true, details: true, emit: false })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'getState', params: { appHandle: selected.appHandle, includeMenus: true, details: true } })
|
||||
await app.getAXState({ includeMenus: false, details: false })
|
||||
await app.getAccessibilityTree({ includeMenus: false, details: false })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'getState', params: { appHandle: selected.appHandle, includeMenus: false, details: false } })
|
||||
})
|
||||
@@ -19,20 +19,25 @@ const snapshot = {
|
||||
}
|
||||
|
||||
it('uses the current tool binding when a retained app is called by another execution', async () => {
|
||||
const first = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => operation.method === 'getApp' ? selected : snapshot)
|
||||
const first = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => {
|
||||
if (operation.method === 'getApp') return selected
|
||||
if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null }
|
||||
return snapshot
|
||||
})
|
||||
let current: CuaToolBinding = first
|
||||
const text = vi.fn()
|
||||
const cua = connectCua(() => current, text)
|
||||
const app = await cua.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
const second = vi.fn(async (): Promise<JsonValue> => ({ dispatched: true }))
|
||||
current = second
|
||||
|
||||
await expect(app.click(element('0'))).resolves.toEqual({ dispatched: true })
|
||||
expect(first).toHaveBeenCalledTimes(2)
|
||||
await expect(window.click(element('0'))).resolves.toEqual({ dispatched: true })
|
||||
expect(first).toHaveBeenCalledTimes(3)
|
||||
expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } })
|
||||
expect(text).not.toHaveBeenCalled()
|
||||
current = first
|
||||
await app.getAXState()
|
||||
await app.getAccessibilityTree()
|
||||
expect(text).toHaveBeenCalledWith('0 button')
|
||||
})
|
||||
|
||||
@@ -57,16 +62,17 @@ it.each([null, 'text only', { content: [{ type: 'text', text: '{}' }] }])('rejec
|
||||
it('returns screenshot bytes and metadata with one image-bearing tool call and no image text output', async () => {
|
||||
const tool = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => {
|
||||
if (operation.method === 'getApp') return selected
|
||||
if (operation.method === 'getState') return snapshot
|
||||
if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null }
|
||||
return {
|
||||
mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 23, bounds, coordinateSpace: 'screenshot-pixels',
|
||||
}
|
||||
})
|
||||
const text = vi.fn()
|
||||
const app = await connectCua(() => tool, text).getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
text.mockClear()
|
||||
|
||||
expect(await app.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
|
||||
expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getApp', 'getState', 'screenshot'])
|
||||
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
|
||||
expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getApp', 'getState', 'getState', 'screenshot'])
|
||||
expect(text).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -1,7 +1,7 @@
|
||||
/** Model-authored operation JSON cannot supply host configuration or bypass retained identities. */
|
||||
import { expect, it } from 'vitest'
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import { Request, ScreenshotResult } from '../src/protocol.ts'
|
||||
import { CapabilitiesResult, Request, ScreenshotResult } from '../src/protocol.ts'
|
||||
|
||||
const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [
|
||||
{ reason: 'raw process ID instead of a selected app handle', request: { method: 'typeText', params: { pid: 17, text: 'hello' } } },
|
||||
@@ -155,3 +155,13 @@ it.each(['click', 'hover', 'scroll', 'drag'])('validates 0–999 integer coordin
|
||||
expect(Request.safeParse({ method, params: { ...params, relative: false, [field]: 1000 } }).success).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it.each([true, false])('requires the reported application-level accessibility capability: %s', (appLevelAccessibility) => {
|
||||
const capabilities = {
|
||||
platform: 'macos', permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility,
|
||||
sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false,
|
||||
}
|
||||
expect(CapabilitiesResult.parse(capabilities)).toEqual(capabilities)
|
||||
const { appLevelAccessibility: _omitted, ...missing } = capabilities
|
||||
expect(CapabilitiesResult.safeParse(missing).success).toBe(false)
|
||||
})
|
||||
@@ -11,6 +11,7 @@ import type { CuaWindowOperation } from '../src/window-types.ts'
|
||||
const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
|
||||
|
||||
const metadata = { windowId: 7, pid: 10, appName: 'Editor', title: 'Document', bounds: { x: -100, y: 0, width: 800, height: 600 }, isOnScreen: true, onCurrentSpace: null }
|
||||
const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC'
|
||||
function fixture() {
|
||||
const text = vi.fn()
|
||||
const dispatch = vi.fn(async (request: CuaWindowOperation): Promise<JsonValue> => {
|
||||
@@ -20,6 +21,8 @@ function fixture() {
|
||||
case 'getState': return { windowHandle: 'window-7', windowId: 7, pid: 10, snapshotId: 'state-1', tree: 'AX window', windows: [metadata], truncated: false }
|
||||
case 'findApps': return [{ id: 'shell-entry', displayName: 'Settings', kind: 'shell' }]
|
||||
case 'listActions': return ['AXPress']
|
||||
case 'hover': return { dispatched: true, hoverRoute: 'accessibility', systemPointerMoved: false }
|
||||
case 'screenshot': return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 7, bounds: metadata.bounds, coordinateSpace: 'screenshot-pixels' }
|
||||
default: return { dispatched: true }
|
||||
}
|
||||
})
|
||||
@@ -39,9 +42,15 @@ it('binds all observation and keyboard operations to a selected window and suppr
|
||||
expect(window).toBeDefined()
|
||||
if (!window) throw new Error('Missing selected window')
|
||||
expect(f.text).not.toHaveBeenCalled()
|
||||
await window.getAXState()
|
||||
expect(f.dispatch.mock.calls[1]?.[0]).toEqual({ method: 'getState', params: { windowHandle: 'window-7' } })
|
||||
await window.getAccessibilityTree()
|
||||
expect(f.text).toHaveBeenCalledWith('AX window')
|
||||
await window.click(element('0'))
|
||||
await window.click([1, 2])
|
||||
await window.hover(element('42.12345'))
|
||||
await window.scroll({ x: 1, y: 2, deltaLinesX: 0, deltaLinesY: 1 })
|
||||
await window.drag([1, 2], [3, 4])
|
||||
expect(await window.getScreenshot()).toMatchObject({ mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 7 })
|
||||
await window.activate()
|
||||
await window.typeText('hello')
|
||||
await window.paste('world')
|
||||
@@ -116,7 +125,7 @@ it('supports discovery without a sink and suppresses requested emissions', async
|
||||
await f.api.listWindows({ emit: false })
|
||||
expect(f.text).not.toHaveBeenCalled()
|
||||
await createCuaWindowApi(f.dispatch).listWindows()
|
||||
const capabilities = { platform: 'windows', permissions: { accessibility: true, screenRecording: true }, sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false }
|
||||
const capabilities = { platform: 'windows', permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: false, sessionLocked: false, methods: [], coordinateSpace: 'screenshot-pixels', browser: false }
|
||||
f.dispatch.mockResolvedValueOnce(capabilities)
|
||||
expect(await f.api.getCapabilities()).toEqual(capabilities)
|
||||
})
|
||||
|
||||
@@ -131,7 +131,7 @@ function createApplicationState(socket, record, application, captureBroker, conn
|
||||
const state = JSON.parse(readFileSync(new URL('./permissions.json', import.meta.url), 'utf8'))
|
||||
return { platform: 'darwin', ...state, helper: { pid: process.pid }, parent: { pid: process.ppid } }
|
||||
}
|
||||
if (method === 'getCapabilities') return { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: true }, methods, coordinateSpace: 'window-points', browser: false }
|
||||
if (method === 'getCapabilities') return { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: true, methods, coordinateSpace: 'window-points', browser: false }
|
||||
if (method === 'listApps') return [{ pid: app.pid, name: app.name, bundleId: app.bundleId }]
|
||||
if (method === 'getApp') {
|
||||
if (params.selector === 'Denied Fixture' || params.selector === 'Revocable Fixture') {
|
||||
|
||||
Reference in new issue
Block a user