diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts new file mode 100644 index 0000000000..2a0a5ed670 --- /dev/null +++ b/packages/experimental/cua/src/api.ts @@ -0,0 +1,151 @@ +/** Object facade over logged native operations; its caller owns evaluator persistence. */ +import { Buffer } from 'node:buffer' +import type { JsonValue } from '@deepseek-ai/dsh-util-values' +import { ActionResult, AppResult, AppsResult, CapabilitiesResult, Request, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts' +import type { CuaApi, CuaApp, CuaAppHandle, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaSurface, CuaWindow } from './types.ts' +export type * from './types.ts' + +function keyChord(chord: string): { key: string; modifiers: string[] } { + const parts = chord.split('+') + const key = parts.pop() + if (!key) throw new Error('A key chord must end with a key name') + const names: Record = { + cmd: 'command', command: 'command', super: 'command', Super_L: 'command', Super_R: 'command', + ctrl: 'control', control: 'control', Control_L: 'control', Control_R: 'control', + alt: 'option', option: 'option', Alt_L: 'option', Alt_R: 'option', + shift: 'shift', Shift_L: 'shift', Shift_R: 'shift', fn: 'fn', + } + return { key, modifiers: parts.map((part) => { + const modifier = names[part] + if (modifier === undefined) throw new Error(`Unsupported key modifier: ${part}`) + return modifier + }) } +} + +/** + * Create app/window objects using the caller's authorized dispatcher. + * @param dispatch - Current tool authority; rejects propagate without retries. + * @param sink - Optional text/image output; emit false suppresses this sink only. + * @returns Objects independent of variable storage and evaluator lifecycle. + */ +export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): CuaApi { + const call = (method: string, params: Record) => { + Request.parse({ method, params }) + return dispatch({ method, params }) + } + const action = async (method: string, params: Record) => ActionResult.parse(await call(method, params)) + async function emit(value: object, options?: CuaObservationOptions): Promise { + if (options?.emit !== false) await sink?.text(JSON.stringify(value)) + } + + function surface(appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface { + let observed: ReturnType | undefined + function snapshotParams(): Record { + if (!observed) throw new Error('Read getAXState before using this surface') + return { appHandle, snapshotId: observed.snapshotId } + } + function windowId(): number { + if (selectedWindow !== undefined) return selectedWindow + const windows = observed?.windows.filter(window => window.isOnScreen && window.bounds.width > 0 && window.bounds.height > 0) + const first = windows?.[0] + if (!first || windows.length !== 1) throw new Error('Select a window with app.getWindows() and app.getWindow(windowId) before this operation') + return first.windowId + } + const result: CuaSurface = { + async getAXState(options) { + observed = undefined + const state = StateResult.parse(await call('getState', { appHandle, ...selectedWindow === undefined ? {} : { windowId: selectedWindow } })) + if (state.appHandle !== appHandle || state.pid !== pid || state.windowId !== (selectedWindow ?? null)) { + throw new Error('Native observation belongs to another application or window') + } + if (state.windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application') + observed = state + if (options?.emit !== false) await sink?.text(state.tree) + return state.tree + }, + async getScreenshot(options) { + if (!observed) await result.getAXState({ emit: false }) + const target = windowId() + const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId: target })) + if (image.windowId !== target) throw new Error('Native screenshot belongs to another window') + const bytes = Buffer.from(image.data, 'base64') + if (bytes.length < 24 || bytes.subarray(0, 8).toString('hex') !== '89504e470d0a1a0a' + || bytes.readUInt32BE(16) !== image.pixelWidth || bytes.readUInt32BE(20) !== image.pixelHeight) { + throw new Error('Native screenshot has invalid PNG dimensions') + } + const png = Uint8Array.from(bytes) + if (options?.emit !== false) await sink?.image(png) + return png + }, + async getAXStateAndScreenshot(options) { + const state = await result.getAXState(options) + return { state, screenshot: await result.getScreenshot(options) } + }, + async click(target, options) { + const mouseButton = options?.mouseButton ?? 'left' + const clickCount = options?.clickCount ?? 1 + if (mouseButton === 'right' && clickCount !== 1) throw new Error('Right double-click is unsupported') + if (typeof target === 'number') { + if (mouseButton !== 'left' || clickCount !== 1) throw new Error('Element clicks support only a single left click') + return action('click', { ...snapshotParams(), elementIndex: target }) + } + const [x, y] = target + return action('click', { ...snapshotParams(), windowId: windowId(), x, y, mouseButton, clickCount }) + }, + async setValue(index, value) { return action('setValue', { ...snapshotParams(), elementIndex: index, value }) }, + async scroll(options) { + if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero') + return action('scroll', { + ...snapshotParams(), windowId: windowId(), + x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY, + }) + }, + async drag(from, to) { + return action('drag', { ...snapshotParams(), windowId: windowId(), fromX: from[0], fromY: from[1], toX: to[0], toY: to[1] }) + }, + } + return result + } + + const api: CuaApi = { + async getCapabilities(options) { + const result = CapabilitiesResult.parse(await call('getCapabilities', {})) + await emit(result, options) + return result + }, + async listApps(options) { + const result = AppsResult.parse(await call('listApps', {})).map(app => ({ + id: app.bundleId || String(app.pid), displayName: app.name, isRunning: true as const, pid: app.pid, + })) + await emit(result, options) + return result + }, + async getState(options) { + const result = { apps: await api.listApps({ emit: false }), browsers: [] as [] } + await emit(result, options) + return result + }, + async getApp(selector) { + const { appHandle, pid } = AppResult.parse(await call('getApp', { selector })) + const app: CuaApp = { + ...surface(appHandle, pid), pid, appHandle, + async getWindows(options) { + const windows = WindowsResult.parse(await call('listWindows', { appHandle })) + if (windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application') + await emit(windows, options) + return windows + }, + async getWindow(windowId): Promise { + const window = { ...surface(appHandle, pid, windowId), windowId } + await window.getAXState() + return window + }, + async typeText(text) { return action('typeText', { appHandle, text }) }, + async pressKey(key) { return action('pressKey', { appHandle, ...keyChord(key) }) }, + } + await app.getAXState() + return app + }, + } + return api +} diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts new file mode 100644 index 0000000000..136cf35931 --- /dev/null +++ b/packages/experimental/cua/tests/api.spec.ts @@ -0,0 +1,331 @@ +/** App and window behavior over a fixture operation dispatcher, without desktop access. */ +import { Buffer } from 'node:buffer' +import { describe, expect, it, vi } from 'vitest' +import type { JsonValue } from '@deepseek-ai/dsh-util-values' +import { createCuaApi } from '../src/api.ts' +import type { CuaApp, CuaOperation } from '../src/types.ts' + +const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC' +const bounds = { x: -640, y: 100, width: 640, height: 480 } +const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor' } +const delivered = { dispatched: true, verified: false } +const capabilities = { platform: 'macos', permissions: { accessibility: true, screenRecording: false }, methods: ['getState', 'click'], coordinateSpace: 'window-points', browser: false } + +function windowInfo(windowId: number) { + return { windowId, pid: 17, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null } +} + +function fixture(withSink = true) { + let windows = [windowInfo(23)] + let generation = 0 + const sink = { text: vi.fn<(value: string) => void>(), image: vi.fn<(data: Uint8Array) => void>() } + const dispatch = vi.fn(async ({ method, params }: CuaOperation): Promise => { + switch (method) { + case 'getCapabilities': return capabilities + case 'listApps': return [selected, { pid: 18, name: 'Utility' }] + case 'getApp': return selected + case 'listWindows': return windows + case 'getState': return { + appHandle: selected.appHandle, pid: 17, windowId: params.windowId ?? null, + snapshotId: `snapshot-${++generation}`, tree: `Tree ${generation}`, truncated: false, + windows: params.windowId === undefined ? windows : windows.filter(window => window.windowId === params.windowId), + } + case 'screenshot': return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: params.windowId!, bounds, coordinateSpace: 'window-points' } + default: return delivered + } + }) + return { + dispatch, sink, api: createCuaApi(dispatch, withSink ? sink : undefined), + setWindows: (ids: number[]) => { windows = ids.map(windowInfo) }, + setWindowRecords: (records: ReturnType[]) => { windows = records }, + } +} + +function state(overrides: Record = {}): JsonValue { + return { appHandle: selected.appHandle, pid: 17, windowId: null, snapshotId: 'changed', tree: 'Changed tree', windows: [windowInfo(23)], truncated: false, ...overrides } +} + +function screenshot(overrides: Record = {}): JsonValue { + return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 23, bounds, coordinateSpace: 'window-points', ...overrides } +} + +describe('application discovery', () => { + it('reports permissions and discovers both bundle and PID selectors without activating an app', async () => { + const f = fixture() + expect(await f.api.getCapabilities()).toEqual(capabilities) + expect(await f.api.getState()).toEqual({ apps: [ + { id: 'test.editor', displayName: 'Editor', isRunning: true, pid: 17 }, + { id: '18', displayName: 'Utility', isRunning: true, pid: 18 }, + ], browsers: [] }) + expect(f.dispatch.mock.calls.map(([operation]) => operation.method)).toEqual(['getCapabilities', 'listApps']) + const emitted = f.sink.text.mock.calls.map(([text]): unknown => JSON.parse(text)) + expect(emitted).toEqual([capabilities, await f.api.getState({ emit: false })]) + expect(f.sink.text).toHaveBeenCalledTimes(2) + }) + + it('returns observations without requiring an output sink', async () => { + const f = fixture(false) + expect(await f.api.getCapabilities({ emit: false })).toEqual(capabilities) + expect(await f.api.listApps()).toHaveLength(2) + const app = await f.api.getApp('Editor') + expect(app.pid).toBe(17) + expect(app.appHandle).toBe('app-17') + expect(await app.getWindows()).toEqual([windowInfo(23)]) + expect(await app.getScreenshot()).toEqual(Uint8Array.from(Buffer.from(PNG, 'base64'))) + expect(f.sink.text).not.toHaveBeenCalled() + expect(f.sink.image).not.toHaveBeenCalled() + }) + + it('rejects malformed discovery and capability results', async () => { + const f = fixture() + f.dispatch.mockResolvedValueOnce({ platform: 'macos' }) + await expect(f.api.getCapabilities()).rejects.toThrow() + f.dispatch.mockResolvedValueOnce([{ pid: 0, name: 'Invalid' }]) + await expect(f.api.listApps()).rejects.toThrow() + f.dispatch.mockResolvedValueOnce({ ...selected, appHandle: '' }) + await expect(f.api.getApp('Editor')).rejects.toThrow() + expect(f.sink.text).not.toHaveBeenCalled() + }) +}) + +describe('application and window observations', () => { + it('keeps the latest AX index scope separate for an app and each selected window', async () => { + const f = fixture() + f.setWindows([23, 24]) + const app = await f.api.getApp('Editor') + expect(await app.getWindows({ emit: false })).toEqual([windowInfo(23), windowInfo(24)]) + const first = await app.getWindow(23) + const second = await app.getWindow(24) + await app.click(0) + await first.setValue(1, 'First document') + await second.click(2) + expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([ + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementIndex: 0 } }, + { method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementIndex: 1, value: 'First document' } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementIndex: 2 } }, + ]) + expect(first.windowId).toBe(23) + expect(second.windowId).toBe(24) + expect(f.sink.text.mock.calls.map(([text]) => text)).toEqual(['Tree 1', 'Tree 2', 'Tree 3']) + await expect(second.getAXState({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4') + await second.click(3) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementIndex: 3 } }) + expect(f.sink.text).toHaveBeenCalledTimes(3) + }) + + it.each([{ ids: [] }, { ids: [23, 24] }])('requires an explicit window when discovery yields $ids', async ({ ids }) => { + const f = fixture() + f.setWindows(ids) + const app = await f.api.getApp('Editor') + const calls = f.dispatch.mock.calls.length + await expect(app.click([1, 2])).rejects.toThrow('Select a window') + await expect(app.getScreenshot()).rejects.toThrow('Select a window') + expect(f.dispatch).toHaveBeenCalledTimes(calls) + }) + + it('selects the one visible nonzero window while preserving menu and hidden windows in discovery', async () => { + const f = fixture() + const windows = [ + windowInfo(23), + { ...windowInfo(24), title: 'Menu helper', bounds: { ...bounds, width: 0, height: 0 } }, + { ...windowInfo(25), isOnScreen: false }, + { ...windowInfo(26), bounds: { ...bounds, height: 0 } }, + ] + f.setWindowRecords(windows) + const app = await f.api.getApp('Editor') + expect(await app.getWindows({ emit: false })).toEqual(windows) + await app.getScreenshot() + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } }) + await app.click([10, 20]) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { + appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1, + } }) + const hidden = await app.getWindow(25) + await hidden.click([10, 20]) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { + appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 25, x: 10, y: 20, mouseButton: 'left', clickCount: 1, + } }) + }) + + it('requires explicit selection when every window is hidden or has zero size', async () => { + const f = fixture() + f.setWindowRecords([ + { ...windowInfo(24), bounds: { ...bounds, width: 0, height: 0 } }, + { ...windowInfo(25), isOnScreen: false }, + ]) + const app = await f.api.getApp('Editor') + const calls = f.dispatch.mock.calls.length + await expect(app.getScreenshot()).rejects.toThrow('Select a window') + expect(f.dispatch).toHaveBeenCalledTimes(calls) + }) + + const identityMismatches: Record[] = [ + { appHandle: 'another-app' }, { pid: 18 }, { windowId: 24 }, + ] + it.each(identityMismatches)('rejects a snapshot belonging to another identity: %j', async (identity) => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockResolvedValueOnce(state(identity)) + await expect(app.getAXState()).rejects.toThrow('another application or window') + const calls = f.dispatch.mock.calls.length + await expect(app.click(0)).rejects.toThrow('Read getAXState') + expect(f.dispatch).toHaveBeenCalledTimes(calls) + }) + + it('rejects wrong-process windows in both snapshot and standalone window discovery', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + const otherWindow = { ...windowInfo(23), pid: 18 } + f.dispatch.mockResolvedValueOnce(state({ windows: [otherWindow] })) + await expect(app.getAXState()).rejects.toThrow('windows belong to another application') + f.dispatch.mockResolvedValueOnce([otherWindow]) + await expect(app.getWindows()).rejects.toThrow('windows belong to another application') + expect(f.sink.text).toHaveBeenCalledTimes(1) + }) + + it('clears a failed refresh and obtains new state before a later screenshot', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + const failure = new Error('Native app is busy') + f.dispatch.mockRejectedValueOnce(failure) + await expect(app.getAXState()).rejects.toBe(failure) + await expect(app.setValue(0, 'stale')).rejects.toThrow('Read getAXState') + f.sink.text.mockClear() + await app.getScreenshot() + expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation.method)).toEqual(['getState', 'screenshot']) + expect(f.sink.text).not.toHaveBeenCalled() + expect(f.sink.image).toHaveBeenCalledTimes(1) + }) +}) + +describe('screenshots', () => { + it('returns PNG pixels independently of input point bounds and supports selective output', async () => { + const f = fixture() + f.setWindows([23, 24]) + const app = await f.api.getApp('Editor') + const window = await app.getWindow(24) + f.sink.text.mockClear() + const observation = await window.getAXStateAndScreenshot() + expect(observation.state).toBe('Tree 3') + expect(observation.screenshot).toEqual(Uint8Array.from(Buffer.from(PNG, 'base64'))) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 24 } }) + expect(f.sink.text).toHaveBeenCalledWith('Tree 3') + expect(f.sink.image).toHaveBeenCalledWith(observation.screenshot) + await window.getAXStateAndScreenshot({ emit: false }) + expect(f.sink.text).toHaveBeenCalledTimes(1) + expect(f.sink.image).toHaveBeenCalledTimes(1) + }) + + it('rejects a screenshot captured from another window before emitting it', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockResolvedValueOnce(screenshot({ windowId: 24 })) + await expect(app.getScreenshot()).rejects.toThrow('another window') + expect(f.sink.image).not.toHaveBeenCalled() + }) + + const invalidImages: Record[] = [ + { data: Buffer.from('short').toString('base64') }, + { data: Buffer.alloc(24).toString('base64') }, + { pixelWidth: 2 }, { pixelHeight: 2 }, + ] + it.each(invalidImages)('rejects invalid PNG dimensions before emitting a screenshot: %j', async (invalid) => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockResolvedValueOnce(screenshot(invalid)) + await expect(app.getScreenshot()).rejects.toThrow('invalid PNG dimensions') + expect(f.sink.image).not.toHaveBeenCalled() + }) +}) + +describe('input actions', () => { + it('keeps the selected app and window when scroll options include another target identity', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + const window = await app.getWindow(23) + const options = { + x: 10, y: 20, deltaLinesX: 0, deltaLinesY: 1, + appHandle: 'another-app', snapshotId: 'another-snapshot', windowId: 24, + } + + await window.scroll(options) + + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'scroll', params: { + appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 10, y: 20, deltaLinesX: 0, deltaLinesY: 1, + } }) + }) + + it('dispatches window-point gestures and reports delivery without claiming the action succeeded', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + expect(await app.click([10, 20], { mouseButton: 'right' })).toEqual(delivered) + expect(await app.click([11, 21], { clickCount: 2 })).toEqual(delivered) + expect(await app.scroll({ x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 })).toEqual(delivered) + expect(await app.drag([1, 2], [30, 40])).toEqual(delivered) + expect(f.dispatch.mock.calls.slice(-4).map(([operation]) => operation)).toEqual([ + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'right', clickCount: 1 } }, + { method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 11, y: 21, mouseButton: 'left', clickCount: 2 } }, + { method: 'scroll', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } }, + { method: 'drag', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, fromX: 1, fromY: 2, toX: 30, toY: 40 } }, + ]) + }) + + const invalidActions: Array<[string, (app: CuaApp) => Promise]> = [ + ['right double click', app => app.click([1, 2], { mouseButton: 'right', clickCount: 2 })], + ['right AX click', app => app.click(0, { mouseButton: 'right' })], + ['double AX click', app => app.click(0, { clickCount: 2 })], + ['negative AX index', app => app.click(-1)], + ['nonfinite coordinate', app => app.click([NaN, 0])], + ['negative coordinate', app => app.click([-1, 0])], + ['negative value index', app => app.setValue(-1, 'value')], + ['zero scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0, deltaLinesY: 0 })], + ['oversized scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })], + ['fractional scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })], + ['negative drag point', app => app.drag([0, 0], [-1, 3])], + ['empty chord', app => app.pressKey('')], + ['missing key', app => app.pressKey('super+')], + ['unknown modifier', app => app.pressKey('unknown+a')], + ] + it.each(invalidActions)('rejects %s without invoking native input', async (_label, run) => { + const f = fixture() + const app = await f.api.getApp('Editor') + const calls = f.dispatch.mock.calls.length + await expect(run(app)).rejects.toThrow() + expect(f.dispatch).toHaveBeenCalledTimes(calls) + }) + + it('passes literal text and keys to the selected app identity', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + await app.typeText('中文\nline two') + await app.pressKey('Control_L+shift+a') + await app.pressKey('Return') + expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([ + { method: 'typeText', params: { appHandle: 'app-17', text: '中文\nline two' } }, + { method: 'pressKey', params: { appHandle: 'app-17', key: 'a', modifiers: ['control', 'shift'] } }, + { method: 'pressKey', params: { appHandle: 'app-17', key: 'Return', modifiers: [] } }, + ]) + }) + + it('propagates native failures once and rejects malformed delivery claims', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + const failure = new Error('Application was restarted; select it again') + f.dispatch.mockRejectedValueOnce(failure) + await expect(app.typeText('hello')).rejects.toBe(failure) + expect(f.dispatch).toHaveBeenCalledTimes(3) + f.dispatch.mockResolvedValueOnce({ dispatched: true, verified: true }) + await expect(app.click(0)).rejects.toThrow() + expect(f.dispatch).toHaveBeenCalledTimes(4) + }) + + it('rejects invalid selection inputs before dispatch', async () => { + const f = fixture() + await expect(f.api.getApp('')).rejects.toThrow() + expect(f.dispatch).not.toHaveBeenCalled() + const app = await f.api.getApp('Editor') + const calls = f.dispatch.mock.calls.length + await expect(app.getWindow(0)).rejects.toThrow() + expect(f.dispatch).toHaveBeenCalledTimes(calls) + }) +})