feat(cua): add scoped application and window objects

Provide discovery, observations, screenshots and input through retained objects. Preserve per-window snapshot identity and prevent action options from replacing the selected target.
This commit is contained in:
creatixchu committed 2026-09-23 16:01:34 +08:00
1 parent b0507f94af
commit 2355950d20
2 files changed
+482

No files matched your search

+151
View File
@@ -0,0 +1,151 @@
/** Object facade over logged native operations; its caller owns evaluator persistence. */
import { Buffer } from 'node:buffer'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import { ActionResult, AppResult, AppsResult, CapabilitiesResult, Request, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts'
import type { CuaApi, CuaApp, CuaAppHandle, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaSurface, CuaWindow } from './types.ts'
export type * from './types.ts'
function keyChord(chord: string): { key: string; modifiers: string[] } {
const parts = chord.split('+')
const key = parts.pop()
if (!key) throw new Error('A key chord must end with a key name')
const names: Record<string, string> = {
cmd: 'command', command: 'command', super: 'command', Super_L: 'command', Super_R: 'command',
ctrl: 'control', control: 'control', Control_L: 'control', Control_R: 'control',
alt: 'option', option: 'option', Alt_L: 'option', Alt_R: 'option',
shift: 'shift', Shift_L: 'shift', Shift_R: 'shift', fn: 'fn',
}
return { key, modifiers: parts.map((part) => {
const modifier = names[part]
if (modifier === undefined) throw new Error(`Unsupported key modifier: ${part}`)
return modifier
}) }
}
/**
* Create app/window objects using the caller's authorized dispatcher.
* @param dispatch - Current tool authority; rejects propagate without retries.
* @param sink - Optional text/image output; emit false suppresses this sink only.
* @returns Objects independent of variable storage and evaluator lifecycle.
*/
export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): CuaApi {
const call = (method: string, params: Record<string, JsonValue>) => {
Request.parse({ method, params })
return dispatch({ method, params })
}
const action = async (method: string, params: Record<string, JsonValue>) => ActionResult.parse(await call(method, params))
async function emit(value: object, options?: CuaObservationOptions): Promise<void> {
if (options?.emit !== false) await sink?.text(JSON.stringify(value))
}
function surface(appHandle: CuaAppHandle, pid: number, selectedWindow?: number): CuaSurface {
let observed: ReturnType<typeof StateResult.parse> | undefined
function snapshotParams(): Record<string, JsonValue> {
if (!observed) throw new Error('Read getAXState before using this surface')
return { appHandle, snapshotId: observed.snapshotId }
}
function windowId(): number {
if (selectedWindow !== undefined) return selectedWindow
const windows = observed?.windows.filter(window => window.isOnScreen && window.bounds.width > 0 && window.bounds.height > 0)
const first = windows?.[0]
if (!first || windows.length !== 1) throw new Error('Select a window with app.getWindows() and app.getWindow(windowId) before this operation')
return first.windowId
}
const result: CuaSurface = {
async getAXState(options) {
observed = undefined
const state = StateResult.parse(await call('getState', { appHandle, ...selectedWindow === undefined ? {} : { windowId: selectedWindow } }))
if (state.appHandle !== appHandle || state.pid !== pid || state.windowId !== (selectedWindow ?? null)) {
throw new Error('Native observation belongs to another application or window')
}
if (state.windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application')
observed = state
if (options?.emit !== false) await sink?.text(state.tree)
return state.tree
},
async getScreenshot(options) {
if (!observed) await result.getAXState({ emit: false })
const target = windowId()
const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId: target }))
if (image.windowId !== target) throw new Error('Native screenshot belongs to another window')
const bytes = Buffer.from(image.data, 'base64')
if (bytes.length < 24 || bytes.subarray(0, 8).toString('hex') !== '89504e470d0a1a0a'
|| bytes.readUInt32BE(16) !== image.pixelWidth || bytes.readUInt32BE(20) !== image.pixelHeight) {
throw new Error('Native screenshot has invalid PNG dimensions')
}
const png = Uint8Array.from(bytes)
if (options?.emit !== false) await sink?.image(png)
return png
},
async getAXStateAndScreenshot(options) {
const state = await result.getAXState(options)
return { state, screenshot: await result.getScreenshot(options) }
},
async click(target, options) {
const mouseButton = options?.mouseButton ?? 'left'
const clickCount = options?.clickCount ?? 1
if (mouseButton === 'right' && clickCount !== 1) throw new Error('Right double-click is unsupported')
if (typeof target === 'number') {
if (mouseButton !== 'left' || clickCount !== 1) throw new Error('Element clicks support only a single left click')
return action('click', { ...snapshotParams(), elementIndex: target })
}
const [x, y] = target
return action('click', { ...snapshotParams(), windowId: windowId(), x, y, mouseButton, clickCount })
},
async setValue(index, value) { return action('setValue', { ...snapshotParams(), elementIndex: index, value }) },
async scroll(options) {
if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero')
return action('scroll', {
...snapshotParams(), windowId: windowId(),
x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY,
})
},
async drag(from, to) {
return action('drag', { ...snapshotParams(), windowId: windowId(), fromX: from[0], fromY: from[1], toX: to[0], toY: to[1] })
},
}
return result
}
const api: CuaApi = {
async getCapabilities(options) {
const result = CapabilitiesResult.parse(await call('getCapabilities', {}))
await emit(result, options)
return result
},
async listApps(options) {
const result = AppsResult.parse(await call('listApps', {})).map(app => ({
id: app.bundleId || String(app.pid), displayName: app.name, isRunning: true as const, pid: app.pid,
}))
await emit(result, options)
return result
},
async getState(options) {
const result = { apps: await api.listApps({ emit: false }), browsers: [] as [] }
await emit(result, options)
return result
},
async getApp(selector) {
const { appHandle, pid } = AppResult.parse(await call('getApp', { selector }))
const app: CuaApp = {
...surface(appHandle, pid), pid, appHandle,
async getWindows(options) {
const windows = WindowsResult.parse(await call('listWindows', { appHandle }))
if (windows.some(window => window.pid !== pid)) throw new Error('Native windows belong to another application')
await emit(windows, options)
return windows
},
async getWindow(windowId): Promise<CuaWindow> {
const window = { ...surface(appHandle, pid, windowId), windowId }
await window.getAXState()
return window
},
async typeText(text) { return action('typeText', { appHandle, text }) },
async pressKey(key) { return action('pressKey', { appHandle, ...keyChord(key) }) },
}
await app.getAXState()
return app
},
}
return api
}
+331
View File
@@ -0,0 +1,331 @@
/** App and window behavior over a fixture operation dispatcher, without desktop access. */
import { Buffer } from 'node:buffer'
import { describe, expect, it, vi } from 'vitest'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import { createCuaApi } from '../src/api.ts'
import type { CuaApp, CuaOperation } from '../src/types.ts'
const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC'
const bounds = { x: -640, y: 100, width: 640, height: 480 }
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor' }
const delivered = { dispatched: true, verified: false }
const capabilities = { platform: 'macos', permissions: { accessibility: true, screenRecording: false }, methods: ['getState', 'click'], coordinateSpace: 'window-points', browser: false }
function windowInfo(windowId: number) {
return { windowId, pid: 17, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null }
}
function fixture(withSink = true) {
let windows = [windowInfo(23)]
let generation = 0
const sink = { text: vi.fn<(value: string) => void>(), image: vi.fn<(data: Uint8Array) => void>() }
const dispatch = vi.fn(async ({ method, params }: CuaOperation): Promise<JsonValue> => {
switch (method) {
case 'getCapabilities': return capabilities
case 'listApps': return [selected, { pid: 18, name: 'Utility' }]
case 'getApp': return selected
case 'listWindows': return windows
case 'getState': return {
appHandle: selected.appHandle, pid: 17, windowId: params.windowId ?? null,
snapshotId: `snapshot-${++generation}`, tree: `Tree ${generation}`, truncated: false,
windows: params.windowId === undefined ? windows : windows.filter(window => window.windowId === params.windowId),
}
case 'screenshot': return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: params.windowId!, bounds, coordinateSpace: 'window-points' }
default: return delivered
}
})
return {
dispatch, sink, api: createCuaApi(dispatch, withSink ? sink : undefined),
setWindows: (ids: number[]) => { windows = ids.map(windowInfo) },
setWindowRecords: (records: ReturnType<typeof windowInfo>[]) => { windows = records },
}
}
function state(overrides: Record<string, JsonValue> = {}): JsonValue {
return { appHandle: selected.appHandle, pid: 17, windowId: null, snapshotId: 'changed', tree: 'Changed tree', windows: [windowInfo(23)], truncated: false, ...overrides }
}
function screenshot(overrides: Record<string, JsonValue> = {}): JsonValue {
return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: 23, bounds, coordinateSpace: 'window-points', ...overrides }
}
describe('application discovery', () => {
it('reports permissions and discovers both bundle and PID selectors without activating an app', async () => {
const f = fixture()
expect(await f.api.getCapabilities()).toEqual(capabilities)
expect(await f.api.getState()).toEqual({ apps: [
{ id: 'test.editor', displayName: 'Editor', isRunning: true, pid: 17 },
{ id: '18', displayName: 'Utility', isRunning: true, pid: 18 },
], browsers: [] })
expect(f.dispatch.mock.calls.map(([operation]) => operation.method)).toEqual(['getCapabilities', 'listApps'])
const emitted = f.sink.text.mock.calls.map(([text]): unknown => JSON.parse(text))
expect(emitted).toEqual([capabilities, await f.api.getState({ emit: false })])
expect(f.sink.text).toHaveBeenCalledTimes(2)
})
it('returns observations without requiring an output sink', async () => {
const f = fixture(false)
expect(await f.api.getCapabilities({ emit: false })).toEqual(capabilities)
expect(await f.api.listApps()).toHaveLength(2)
const app = await f.api.getApp('Editor')
expect(app.pid).toBe(17)
expect(app.appHandle).toBe('app-17')
expect(await app.getWindows()).toEqual([windowInfo(23)])
expect(await app.getScreenshot()).toEqual(Uint8Array.from(Buffer.from(PNG, 'base64')))
expect(f.sink.text).not.toHaveBeenCalled()
expect(f.sink.image).not.toHaveBeenCalled()
})
it('rejects malformed discovery and capability results', async () => {
const f = fixture()
f.dispatch.mockResolvedValueOnce({ platform: 'macos' })
await expect(f.api.getCapabilities()).rejects.toThrow()
f.dispatch.mockResolvedValueOnce([{ pid: 0, name: 'Invalid' }])
await expect(f.api.listApps()).rejects.toThrow()
f.dispatch.mockResolvedValueOnce({ ...selected, appHandle: '' })
await expect(f.api.getApp('Editor')).rejects.toThrow()
expect(f.sink.text).not.toHaveBeenCalled()
})
})
describe('application and window observations', () => {
it('keeps the latest AX index scope separate for an app and each selected window', async () => {
const f = fixture()
f.setWindows([23, 24])
const app = await f.api.getApp('Editor')
expect(await app.getWindows({ emit: false })).toEqual([windowInfo(23), windowInfo(24)])
const first = await app.getWindow(23)
const second = await app.getWindow(24)
await app.click(0)
await first.setValue(1, 'First document')
await second.click(2)
expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementIndex: 0 } },
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementIndex: 1, value: 'First document' } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementIndex: 2 } },
])
expect(first.windowId).toBe(23)
expect(second.windowId).toBe(24)
expect(f.sink.text.mock.calls.map(([text]) => text)).toEqual(['Tree 1', 'Tree 2', 'Tree 3'])
await expect(second.getAXState({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4')
await second.click(3)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementIndex: 3 } })
expect(f.sink.text).toHaveBeenCalledTimes(3)
})
it.each([{ ids: [] }, { ids: [23, 24] }])('requires an explicit window when discovery yields $ids', async ({ ids }) => {
const f = fixture()
f.setWindows(ids)
const app = await f.api.getApp('Editor')
const calls = f.dispatch.mock.calls.length
await expect(app.click([1, 2])).rejects.toThrow('Select a window')
await expect(app.getScreenshot()).rejects.toThrow('Select a window')
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
it('selects the one visible nonzero window while preserving menu and hidden windows in discovery', async () => {
const f = fixture()
const windows = [
windowInfo(23),
{ ...windowInfo(24), title: 'Menu helper', bounds: { ...bounds, width: 0, height: 0 } },
{ ...windowInfo(25), isOnScreen: false },
{ ...windowInfo(26), bounds: { ...bounds, height: 0 } },
]
f.setWindowRecords(windows)
const app = await f.api.getApp('Editor')
expect(await app.getWindows({ emit: false })).toEqual(windows)
await app.getScreenshot()
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } })
await app.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
const hidden = await app.getWindow(25)
await hidden.click([10, 20])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 25, x: 10, y: 20, mouseButton: 'left', clickCount: 1,
} })
})
it('requires explicit selection when every window is hidden or has zero size', async () => {
const f = fixture()
f.setWindowRecords([
{ ...windowInfo(24), bounds: { ...bounds, width: 0, height: 0 } },
{ ...windowInfo(25), isOnScreen: false },
])
const app = await f.api.getApp('Editor')
const calls = f.dispatch.mock.calls.length
await expect(app.getScreenshot()).rejects.toThrow('Select a window')
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
const identityMismatches: Record<string, JsonValue>[] = [
{ appHandle: 'another-app' }, { pid: 18 }, { windowId: 24 },
]
it.each(identityMismatches)('rejects a snapshot belonging to another identity: %j', async (identity) => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockResolvedValueOnce(state(identity))
await expect(app.getAXState()).rejects.toThrow('another application or window')
const calls = f.dispatch.mock.calls.length
await expect(app.click(0)).rejects.toThrow('Read getAXState')
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
it('rejects wrong-process windows in both snapshot and standalone window discovery', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const otherWindow = { ...windowInfo(23), pid: 18 }
f.dispatch.mockResolvedValueOnce(state({ windows: [otherWindow] }))
await expect(app.getAXState()).rejects.toThrow('windows belong to another application')
f.dispatch.mockResolvedValueOnce([otherWindow])
await expect(app.getWindows()).rejects.toThrow('windows belong to another application')
expect(f.sink.text).toHaveBeenCalledTimes(1)
})
it('clears a failed refresh and obtains new state before a later screenshot', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const failure = new Error('Native app is busy')
f.dispatch.mockRejectedValueOnce(failure)
await expect(app.getAXState()).rejects.toBe(failure)
await expect(app.setValue(0, 'stale')).rejects.toThrow('Read getAXState')
f.sink.text.mockClear()
await app.getScreenshot()
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation.method)).toEqual(['getState', 'screenshot'])
expect(f.sink.text).not.toHaveBeenCalled()
expect(f.sink.image).toHaveBeenCalledTimes(1)
})
})
describe('screenshots', () => {
it('returns PNG pixels independently of input point bounds and supports selective output', async () => {
const f = fixture()
f.setWindows([23, 24])
const app = await f.api.getApp('Editor')
const window = await app.getWindow(24)
f.sink.text.mockClear()
const observation = await window.getAXStateAndScreenshot()
expect(observation.state).toBe('Tree 3')
expect(observation.screenshot).toEqual(Uint8Array.from(Buffer.from(PNG, 'base64')))
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'screenshot', params: { appHandle: 'app-17', windowId: 24 } })
expect(f.sink.text).toHaveBeenCalledWith('Tree 3')
expect(f.sink.image).toHaveBeenCalledWith(observation.screenshot)
await window.getAXStateAndScreenshot({ emit: false })
expect(f.sink.text).toHaveBeenCalledTimes(1)
expect(f.sink.image).toHaveBeenCalledTimes(1)
})
it('rejects a screenshot captured from another window before emitting it', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockResolvedValueOnce(screenshot({ windowId: 24 }))
await expect(app.getScreenshot()).rejects.toThrow('another window')
expect(f.sink.image).not.toHaveBeenCalled()
})
const invalidImages: Record<string, JsonValue>[] = [
{ data: Buffer.from('short').toString('base64') },
{ data: Buffer.alloc(24).toString('base64') },
{ pixelWidth: 2 }, { pixelHeight: 2 },
]
it.each(invalidImages)('rejects invalid PNG dimensions before emitting a screenshot: %j', async (invalid) => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockResolvedValueOnce(screenshot(invalid))
await expect(app.getScreenshot()).rejects.toThrow('invalid PNG dimensions')
expect(f.sink.image).not.toHaveBeenCalled()
})
})
describe('input actions', () => {
it('keeps the selected app and window when scroll options include another target identity', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const window = await app.getWindow(23)
const options = {
x: 10, y: 20, deltaLinesX: 0, deltaLinesY: 1,
appHandle: 'another-app', snapshotId: 'another-snapshot', windowId: 24,
}
await window.scroll(options)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'scroll', params: {
appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 10, y: 20, deltaLinesX: 0, deltaLinesY: 1,
} })
})
it('dispatches window-point gestures and reports delivery without claiming the action succeeded', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
expect(await app.click([10, 20], { mouseButton: 'right' })).toEqual(delivered)
expect(await app.click([11, 21], { clickCount: 2 })).toEqual(delivered)
expect(await app.scroll({ x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 })).toEqual(delivered)
expect(await app.drag([1, 2], [30, 40])).toEqual(delivered)
expect(f.dispatch.mock.calls.slice(-4).map(([operation]) => operation)).toEqual([
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, mouseButton: 'right', clickCount: 1 } },
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 11, y: 21, mouseButton: 'left', clickCount: 2 } },
{ method: 'scroll', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, x: 10, y: 20, deltaLinesX: -2, deltaLinesY: 3 } },
{ method: 'drag', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', windowId: 23, fromX: 1, fromY: 2, toX: 30, toY: 40 } },
])
})
const invalidActions: Array<[string, (app: CuaApp) => Promise<unknown>]> = [
['right double click', app => app.click([1, 2], { mouseButton: 'right', clickCount: 2 })],
['right AX click', app => app.click(0, { mouseButton: 'right' })],
['double AX click', app => app.click(0, { clickCount: 2 })],
['negative AX index', app => app.click(-1)],
['nonfinite coordinate', app => app.click([NaN, 0])],
['negative coordinate', app => app.click([-1, 0])],
['negative value index', app => app.setValue(-1, 'value')],
['zero scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0, deltaLinesY: 0 })],
['oversized scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })],
['fractional scroll', app => app.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })],
['negative drag point', app => app.drag([0, 0], [-1, 3])],
['empty chord', app => app.pressKey('')],
['missing key', app => app.pressKey('super+')],
['unknown modifier', app => app.pressKey('unknown+a')],
]
it.each(invalidActions)('rejects %s without invoking native input', async (_label, run) => {
const f = fixture()
const app = await f.api.getApp('Editor')
const calls = f.dispatch.mock.calls.length
await expect(run(app)).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
it('passes literal text and keys to the selected app identity', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
await app.typeText('中文\nline two')
await app.pressKey('Control_L+shift+a')
await app.pressKey('Return')
expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([
{ method: 'typeText', params: { appHandle: 'app-17', text: '中文\nline two' } },
{ method: 'pressKey', params: { appHandle: 'app-17', key: 'a', modifiers: ['control', 'shift'] } },
{ method: 'pressKey', params: { appHandle: 'app-17', key: 'Return', modifiers: [] } },
])
})
it('propagates native failures once and rejects malformed delivery claims', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const failure = new Error('Application was restarted; select it again')
f.dispatch.mockRejectedValueOnce(failure)
await expect(app.typeText('hello')).rejects.toBe(failure)
expect(f.dispatch).toHaveBeenCalledTimes(3)
f.dispatch.mockResolvedValueOnce({ dispatched: true, verified: true })
await expect(app.click(0)).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(4)
})
it('rejects invalid selection inputs before dispatch', async () => {
const f = fixture()
await expect(f.api.getApp('')).rejects.toThrow()
expect(f.dispatch).not.toHaveBeenCalled()
const app = await f.api.getApp('Editor')
const calls = f.dispatch.mock.calls.length
await expect(app.getWindow(0)).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
})