refactor(cua): centralize scoped surfaces and provider capabilities
This commit is contained in:
1 parent
9aa3272fa5
commit
ca0760bd1c
17 files changed
+317
-121
No files matched your search
@@ -191,12 +191,9 @@ actor AXWorker {
|
||||
guard !closed else { throw NativeFailure(code: "NATIVE_CLOSED", message: "Helper is closing") }
|
||||
let locked = observeSession()
|
||||
if method == "getCapabilities" {
|
||||
return .result(.object(["platform": .string("macos"), "permissions": .object([
|
||||
return .result(.object(["permissions": .object([
|
||||
"accessibility": .bool(AXIsProcessTrusted()), "screenRecording": .bool(CGPreflightScreenCaptureAccess())]),
|
||||
"appLevelAccessibility": .bool(true),
|
||||
"keyboard": .array([.string("app")]),
|
||||
"sessionLocked": locked.map(JSONValue.bool) ?? .null,
|
||||
"methods": .array(NativeProtocol.modelMethods.map(JSONValue.string)), "coordinateSpace": .string("window-points"), "browser": .bool(false)]))
|
||||
"sessionLocked": locked.map(JSONValue.bool) ?? .null]))
|
||||
}
|
||||
if method == "listApps" { return .result(.array(catalog.applications.filter { $0.regular && $0.name != nil }.map(\.json))) }
|
||||
try NativePlatform.requireUnlocked(locked)
|
||||
|
||||
+8
-9
@@ -1,6 +1,5 @@
|
||||
// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.
|
||||
enum NativeProtocol {
|
||||
static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "invoke", "listActions", "listApps", "listWindows", "paste", "performAction", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"]
|
||||
static let fields: [String: Set<String>] = [
|
||||
"activate": ["appHandle"],
|
||||
"applicationIcons": ["paths"],
|
||||
@@ -11,23 +10,23 @@ enum NativeProtocol {
|
||||
"getCapabilities": [],
|
||||
"getState": ["appHandle", "details", "includeMenus", "maxDepth", "maxElements", "windowId"],
|
||||
"hover": ["appHandle", "elementId", "snapshotId", "windowId", "x", "y"],
|
||||
"invoke": ["appHandle", "elementId", "snapshotId"],
|
||||
"invoke": ["appHandle", "elementId", "snapshotId", "windowId"],
|
||||
"launchApp": ["appHandle"],
|
||||
"listActions": ["appHandle", "elementId", "snapshotId"],
|
||||
"listActions": ["appHandle", "elementId", "snapshotId", "windowId"],
|
||||
"listApps": [],
|
||||
"listWindows": ["appHandle"],
|
||||
"openPermissionSettings": ["permission"],
|
||||
"paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text"],
|
||||
"performAction": ["action", "appHandle", "argument", "elementId", "snapshotId"],
|
||||
"paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text", "windowId"],
|
||||
"performAction": ["action", "appHandle", "argument", "elementId", "snapshotId", "windowId"],
|
||||
"permissionStatus": [],
|
||||
"pressKey": ["appHandle", "key", "modifiers"],
|
||||
"pressKey": ["appHandle", "key", "modifiers", "windowId"],
|
||||
"requestPermission": ["permission"],
|
||||
"screenshot": ["appHandle", "windowId"],
|
||||
"scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"],
|
||||
"selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text"],
|
||||
"setValue": ["appHandle", "elementId", "snapshotId", "value"],
|
||||
"selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text", "windowId"],
|
||||
"setValue": ["appHandle", "elementId", "snapshotId", "value", "windowId"],
|
||||
"show": ["appHandle", "windowId"],
|
||||
"typeText": ["appHandle", "text"],
|
||||
"typeText": ["appHandle", "text", "windowId"],
|
||||
]
|
||||
static let stringEnums: [String: [String: Set<String>]] = [
|
||||
"click": ["modifiers": ["command", "control", "fn", "option", "shift"], "mouseButton": ["left", "middle", "right"]],
|
||||
|
||||
@@ -7,12 +7,11 @@ import { nativeProtocol } from '@deepseek-ai/dsh-cua/native-schema'
|
||||
* @returns Native JSON Schema and Swift field/enum tables, including private continuations.
|
||||
*/
|
||||
export function renderNativeProtocol(): { json: string; swift: string } {
|
||||
const { fields, enums, modelMethods, json } = nativeProtocol()
|
||||
const { fields, enums, json } = nativeProtocol()
|
||||
const quoted = (values: Iterable<string>) => [...values].sort().map(value => JSON.stringify(value)).join(', ')
|
||||
const swift = [
|
||||
'// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.',
|
||||
'enum NativeProtocol {',
|
||||
` static let modelMethods: [String] = [${quoted(modelMethods)}]`,
|
||||
' static let fields: [String: Set<String>] = [',
|
||||
...[...fields].sort(([a], [b]) => a.localeCompare(b)).map(([method, names]) => ` ${JSON.stringify(method)}: [${quoted(names)}],`),
|
||||
' ]',
|
||||
|
||||
@@ -290,6 +290,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
@@ -332,6 +337,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
@@ -415,6 +425,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
@@ -459,6 +474,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
@@ -538,6 +558,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
@@ -693,6 +718,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
@@ -772,6 +802,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
@@ -833,6 +868,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
}
|
||||
@@ -868,6 +908,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"text": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -934,6 +979,11 @@
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"key": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
|
||||
@@ -144,7 +144,6 @@ def main():
|
||||
call = helper.call
|
||||
capabilities = call("getCapabilities", {})
|
||||
assert capabilities["permissions"] == {"accessibility": True, "screenRecording": True}, capabilities
|
||||
assert "launchApp" not in capabilities["methods"], capabilities
|
||||
matches = call("findApps", {"query": str(application), "maxAppMatches": 20})
|
||||
assert len(matches) == 1 and matches[0]["isRunning"] is False, matches
|
||||
pending = call("getApp", {"selector": str(application)})
|
||||
|
||||
@@ -126,7 +126,6 @@ def main():
|
||||
time.sleep(0.025)
|
||||
|
||||
capabilities = call("getCapabilities", {})
|
||||
assert capabilities["platform"] == "macos", capabilities
|
||||
assert capabilities["permissions"] == {"accessibility": True, "screenRecording": True}, capabilities
|
||||
apps = call("listApps", {})
|
||||
assert any(app["pid"] == fixture.pid for app in apps), apps
|
||||
|
||||
@@ -3,11 +3,25 @@ import { isAbsolute, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import type { Context } from '@deepseek-ai/cordis'
|
||||
import { CuaConfig, installCua } from '@deepseek-ai/dsh-cua'
|
||||
import type { CuaStartupCapabilities } from '@deepseek-ai/dsh-cua'
|
||||
import { nativeHostArchitecture, prepareNativeExecutable } from './native-build.ts'
|
||||
import { createSharedNativeChannel } from './shared-native.ts'
|
||||
import { GUIDANCE } from './guidance.ts'
|
||||
export { GUIDANCE } from './guidance.ts'
|
||||
|
||||
const CAPABILITIES: CuaStartupCapabilities = {
|
||||
platform: 'macos',
|
||||
methods: [
|
||||
'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'hover', 'invoke',
|
||||
'listActions', 'listApps', 'listWindows', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll',
|
||||
'selectText', 'setValue', 'show', 'typeText',
|
||||
],
|
||||
appLevelAccessibility: true,
|
||||
keyboard: ['app'],
|
||||
browser: false,
|
||||
coordinateSpace: 'screenshot-pixels',
|
||||
}
|
||||
|
||||
/** Loader plugin identity. */
|
||||
export const name = 'experimental-cua-macos'
|
||||
/** Services required by the shared CUA executor and the native launcher. */
|
||||
@@ -27,7 +41,9 @@ export const Config: typeof CuaConfig = CuaConfig
|
||||
export async function apply(ctx: Context, config: Config): Promise<void> {
|
||||
if (config.helperPath !== undefined) {
|
||||
if (!isAbsolute(config.helperPath)) throw new Error('dsh-cua helperPath must be absolute')
|
||||
installCua(ctx, config, { helperPath: config.helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE })
|
||||
installCua(ctx, config, {
|
||||
helperPath: config.helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE, capabilities: CAPABILITIES,
|
||||
})
|
||||
return
|
||||
}
|
||||
const architecture = nativeHostArchitecture(process.platform, process.arch)
|
||||
@@ -44,5 +60,5 @@ export async function apply(ctx: Context, config: Config): Promise<void> {
|
||||
}, 'dsh-cua.native-preparation')
|
||||
const helperPath = await prepared
|
||||
controller.signal.throwIfAborted()
|
||||
installCua(ctx, config, { helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE })
|
||||
installCua(ctx, config, { helperPath, createChannel: createSharedNativeChannel, guidance: GUIDANCE, capabilities: CAPABILITIES })
|
||||
}
|
||||
@@ -162,13 +162,18 @@ it('correlates a native request with its logged tool invocation', async () => {
|
||||
expect(native.request).toHaveBeenCalledExactlyOnceWith('listApps', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1'))
|
||||
})
|
||||
|
||||
it('reports screenshot pixel coordinates while preserving native capabilities', async () => {
|
||||
it('merges startup capabilities into runtime native fields', async () => {
|
||||
const f = await setup()
|
||||
const capabilities = { platform: 'macos', coordinateSpace: 'window-points', permissions: { accessibility: true, screenCapture: false }, appLevelAccessibility: true, keyboard: ['app'] }
|
||||
native.request.mockResolvedValueOnce(capabilities)
|
||||
const runtime = { permissions: { accessibility: true, screenRecording: false }, sessionLocked: false }
|
||||
native.request.mockResolvedValueOnce(runtime)
|
||||
const result = await f.execute('getCapabilities')
|
||||
expect(result.isError).toBe(false)
|
||||
expect(result.value).toEqual({ ...capabilities, coordinateSpace: 'screenshot-pixels' })
|
||||
expect(result.value).toEqual({ ...runtime, platform: 'macos', appLevelAccessibility: true, keyboard: ['app'], browser: false,
|
||||
coordinateSpace: 'screenshot-pixels', methods: [
|
||||
'activate', 'click', 'drag', 'findApps', 'getApp', 'getCapabilities', 'getState', 'hover', 'invoke',
|
||||
'listActions', 'listApps', 'listWindows', 'paste', 'performAction', 'pressKey', 'screenshot', 'scroll',
|
||||
'selectText', 'setValue', 'show', 'typeText',
|
||||
] })
|
||||
expect(native.request).toHaveBeenCalledExactlyOnceWith('getCapabilities', {}, expect.any(AbortSignal), ToolCallId('lifecycle-1'))
|
||||
})
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import { Buffer } from 'node:buffer'
|
||||
import type { z } from 'zod'
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts'
|
||||
import type { CuaApi, CuaApp, CuaAppHandle, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts'
|
||||
import type { CuaApi, CuaApp, CuaAppHandle, CuaCapabilities, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts'
|
||||
export type * from './types.ts'
|
||||
|
||||
function requestIssueMessage(issues: readonly z.core.$ZodIssue[], parent: readonly PropertyKey[] = []): string {
|
||||
@@ -56,10 +56,6 @@ function validatedCall(dispatch: CuaDispatch) {
|
||||
}
|
||||
}
|
||||
|
||||
async function unsupportedKeyboardInput(): Promise<never> {
|
||||
throw new Error('Keyboard input is not supported at this scope.')
|
||||
}
|
||||
|
||||
async function emit(sink: CuaObservationSink | undefined, value: object, options?: CuaObservationOptions): Promise<void> {
|
||||
if (options?.emit !== false) await sink?.text(JSON.stringify(value))
|
||||
}
|
||||
@@ -72,11 +68,17 @@ async function emit(sink: CuaObservationSink | undefined, value: object, options
|
||||
*/
|
||||
export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink): CuaApi {
|
||||
const call = validatedCall(dispatch)
|
||||
const objects = createCuaObjects(dispatch, sink)
|
||||
let capabilities: CuaCapabilities | undefined
|
||||
async function currentCapabilities(): Promise<CuaCapabilities> {
|
||||
if (capabilities === undefined) capabilities = CapabilitiesResult.parse(await call('getCapabilities', {}))
|
||||
return capabilities
|
||||
}
|
||||
const objects = createCuaObjects(call, sink)
|
||||
|
||||
const api: CuaApi = {
|
||||
async getCapabilities(options) {
|
||||
const result = CapabilitiesResult.parse(await call('getCapabilities', {}))
|
||||
capabilities = result
|
||||
await emit(sink, result, options)
|
||||
return result
|
||||
},
|
||||
@@ -98,82 +100,112 @@ export function createCuaApi(dispatch: CuaDispatch, sink?: CuaObservationSink):
|
||||
return result
|
||||
},
|
||||
async getApp(selector) {
|
||||
const request = Request.safeParse({ method: 'getApp', params: { selector } })
|
||||
if (!request.success) throw new Error(`getApp ${requestIssueMessage(request.error.issues)}`)
|
||||
const capabilities = await currentCapabilities()
|
||||
const { appHandle } = AppResult.parse(await call('getApp', { selector }))
|
||||
const app = objects.createApp(appHandle)
|
||||
await app.getAccessibilityTree({ emit: false })
|
||||
const app = objects.createCuaApp(appHandle, capabilities)
|
||||
if (capabilities.appLevelAccessibility) await app.getAccessibilityTree({ emit: false })
|
||||
return app
|
||||
},
|
||||
}
|
||||
return api
|
||||
}
|
||||
|
||||
function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | undefined): {
|
||||
createApp(appHandle: CuaAppHandle): CuaApp
|
||||
createWindow(appHandle: CuaAppHandle, windowId: number): CuaWindow
|
||||
type CuaScopeParams = (params: Record<string, JsonValue>) => Record<string, JsonValue>
|
||||
|
||||
function createCuaObjects(call: ReturnType<typeof validatedCall>, sink: CuaObservationSink | undefined): {
|
||||
createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp
|
||||
} {
|
||||
const call = validatedCall(dispatch)
|
||||
const action = async (method: CuaOperation['method'], params: Record<string, JsonValue>) => ActionResult.parse(await call(method, params))
|
||||
const observed = new Map<CuaAppHandle, { snapshotId: string; windowId: number | undefined }>()
|
||||
function elementParams(appHandle: CuaAppHandle): Record<string, JsonValue> {
|
||||
function currentSnapshot(appHandle: CuaAppHandle): string {
|
||||
const current = observed.get(appHandle)
|
||||
if (current === undefined) throw new Error('Read getAccessibilityTree before using this surface')
|
||||
return { appHandle, snapshotId: current.snapshotId }
|
||||
return current.snapshotId
|
||||
}
|
||||
function coordinateParams(appHandle: CuaAppHandle, windowId: number): Record<string, JsonValue> {
|
||||
const current = observed.get(appHandle)
|
||||
return {
|
||||
appHandle, windowId,
|
||||
...current !== undefined && (current.windowId === undefined || current.windowId === windowId)
|
||||
? { snapshotId: current.snapshotId } : {},
|
||||
}
|
||||
return current !== undefined && (current.windowId === undefined || current.windowId === windowId)
|
||||
? { snapshotId: current.snapshotId } : {}
|
||||
}
|
||||
function createSurface(appHandle: CuaAppHandle, selectedWindow?: number): CuaSurface {
|
||||
return {
|
||||
async show() {
|
||||
return ShowResult.parse(await call('show', { appHandle,
|
||||
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
|
||||
}))
|
||||
},
|
||||
function createCuaSurface(
|
||||
target: { appHandle: CuaAppHandle; windowId?: number },
|
||||
paramsFor: CuaScopeParams,
|
||||
requireAccessibility: () => void,
|
||||
requireKeyboard: () => void,
|
||||
) {
|
||||
const scopedCall = (method: CuaOperation['method'], params: Record<string, JsonValue>) =>
|
||||
call(method, paramsFor(params))
|
||||
const action = async (method: CuaOperation['method'], params: Record<string, JsonValue>) =>
|
||||
ActionResult.parse(await scopedCall(method, params))
|
||||
const elementParams = (): Record<string, JsonValue> => ({ snapshotId: currentSnapshot(target.appHandle) })
|
||||
const surface: CuaSurface = {
|
||||
async show() { return ShowResult.parse(await scopedCall('show', {})) },
|
||||
async getAccessibilityTree(options) {
|
||||
observed.delete(appHandle)
|
||||
const state = StateResult.parse(await call('getState', { appHandle,
|
||||
...selectedWindow === undefined ? {} : { windowId: selectedWindow },
|
||||
requireAccessibility()
|
||||
observed.delete(target.appHandle)
|
||||
const state = StateResult.parse(await scopedCall('getState', {
|
||||
...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus },
|
||||
...options?.details === undefined ? {} : { details: options.details },
|
||||
}))
|
||||
observed.set(appHandle, { snapshotId: state.snapshotId, windowId: selectedWindow })
|
||||
observed.set(target.appHandle, { snapshotId: state.snapshotId, windowId: target.windowId })
|
||||
if (options?.emit !== false) await sink?.text(state.tree)
|
||||
return state.tree
|
||||
},
|
||||
async listActions(elementId) {
|
||||
return ActionNamesResult.parse(await call('listActions', { ...elementParams(appHandle), elementId }))
|
||||
requireAccessibility()
|
||||
return ActionNamesResult.parse(await scopedCall('listActions', { ...elementParams(), elementId }))
|
||||
},
|
||||
async performAction(elementId, name, argument) {
|
||||
return action('performAction', { ...elementParams(appHandle), elementId, action: name,
|
||||
requireAccessibility()
|
||||
return action('performAction', { ...elementParams(), elementId, action: name,
|
||||
...argument === undefined ? {} : { argument },
|
||||
})
|
||||
},
|
||||
async invoke(elementId) {
|
||||
return ActionResult.parse(await call('invoke', { ...elementParams(appHandle), elementId }))
|
||||
requireAccessibility()
|
||||
return action('invoke', { ...elementParams(), elementId })
|
||||
},
|
||||
async setValue(elementId, value) {
|
||||
requireAccessibility()
|
||||
return action('setValue', { ...elementParams(), elementId, value })
|
||||
},
|
||||
async setValue(elementId, value) { return action('setValue', { ...elementParams(appHandle), elementId, value }) },
|
||||
async selectText(elementId, text, options) {
|
||||
return action('selectText', { ...elementParams(appHandle), elementId, text,
|
||||
requireAccessibility()
|
||||
return action('selectText', { ...elementParams(), elementId, text,
|
||||
...options?.prefix === undefined ? {} : { prefix: options.prefix },
|
||||
...options?.suffix === undefined ? {} : { suffix: options.suffix },
|
||||
...options?.selectionType === undefined ? {} : { selectionType: options.selectionType },
|
||||
})
|
||||
},
|
||||
typeText: unsupportedKeyboardInput,
|
||||
paste: unsupportedKeyboardInput,
|
||||
pressKey: unsupportedKeyboardInput,
|
||||
async typeText(text) {
|
||||
requireKeyboard()
|
||||
return action('typeText', { text })
|
||||
},
|
||||
async paste(text, options) {
|
||||
requireKeyboard()
|
||||
return action('paste', { text,
|
||||
...options?.format === undefined ? {} : { format: options.format },
|
||||
})
|
||||
},
|
||||
async pressKey(key) {
|
||||
requireKeyboard()
|
||||
return action('pressKey', keyChord(key))
|
||||
},
|
||||
}
|
||||
return { surface, call: scopedCall, action, elementParams }
|
||||
}
|
||||
function createWindow(appHandle: CuaAppHandle, windowId: number): CuaWindow {
|
||||
function createCuaWindow(appHandle: CuaAppHandle, windowId: number, capabilities: CuaCapabilities): CuaWindow {
|
||||
const base = createCuaSurface({ appHandle, windowId },
|
||||
params => ({ ...params, appHandle, windowId }), () => undefined, () => {
|
||||
if (!capabilities.keyboard.includes('window')) {
|
||||
throw new Error('Window-scoped keyboard input is not supported on current platform.')
|
||||
}
|
||||
})
|
||||
return {
|
||||
...createSurface(appHandle, windowId), windowId,
|
||||
...base.surface, windowId,
|
||||
async getScreenshot(options) {
|
||||
const image = ScreenshotResult.parse(await call('screenshot', { appHandle, windowId }))
|
||||
const image = ScreenshotResult.parse(await base.call('screenshot', {}))
|
||||
const screenshot: CuaScreenshot = {
|
||||
bytes: Uint8Array.from(Buffer.from(image.data, 'base64')),
|
||||
mimeType: image.mimeType,
|
||||
@@ -184,37 +216,39 @@ function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | unde
|
||||
if (options?.emit !== false) await sink?.image(screenshot)
|
||||
return screenshot
|
||||
},
|
||||
async hover(target, options) {
|
||||
async hover(targetPoint, options) {
|
||||
const relative = options?.relative === undefined ? {} : { relative: options.relative }
|
||||
if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...elementParams(appHandle), elementId: target, ...relative }))
|
||||
const [x, y] = target
|
||||
return HoverResult.parse(await call('hover', { ...coordinateParams(appHandle, windowId), x, y, ...relative }))
|
||||
if (typeof targetPoint === 'string') {
|
||||
return HoverResult.parse(await base.call('hover', { ...base.elementParams(), elementId: targetPoint, ...relative }))
|
||||
}
|
||||
const [x, y] = targetPoint
|
||||
return HoverResult.parse(await base.call('hover', { ...coordinateParams(appHandle, windowId), x, y, ...relative }))
|
||||
},
|
||||
async click(target, options) {
|
||||
async click(targetPoint, options) {
|
||||
const mouseButton = options?.mouseButton ?? 'left'
|
||||
const clickCount = options?.clickCount ?? 1
|
||||
if (mouseButton !== 'left' && clickCount !== 1) throw new Error('Only left double-click is supported')
|
||||
const modifiers = options?.modifiers === undefined ? {} : { modifiers: [...options.modifiers] }
|
||||
const hold = options?.holdMs === undefined ? {} : { holdMs: options.holdMs }
|
||||
const relative = options?.relative === undefined ? {} : { relative: options.relative }
|
||||
if (typeof target === 'string') {
|
||||
return action('click', { ...elementParams(appHandle), ...modifiers, ...hold, ...relative, elementId: target,
|
||||
if (typeof targetPoint === 'string') {
|
||||
return base.action('click', { ...base.elementParams(), ...modifiers, ...hold, ...relative, elementId: targetPoint,
|
||||
...mouseButton === 'left' ? {} : { mouseButton }, ...clickCount === 1 ? {} : { clickCount },
|
||||
})
|
||||
}
|
||||
const [x, y] = target
|
||||
return action('click', { ...coordinateParams(appHandle, windowId), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative })
|
||||
const [x, y] = targetPoint
|
||||
return base.action('click', { ...coordinateParams(appHandle, windowId), x, y, mouseButton, clickCount, ...modifiers, ...hold, ...relative })
|
||||
},
|
||||
async scroll(options) {
|
||||
if (options.deltaLinesX === 0 && options.deltaLinesY === 0) throw new Error('At least one scroll delta must be nonzero')
|
||||
return action('scroll', {
|
||||
return base.action('scroll', {
|
||||
...coordinateParams(appHandle, windowId),
|
||||
x: options.x, y: options.y, deltaLinesX: options.deltaLinesX, deltaLinesY: options.deltaLinesY,
|
||||
...options.relative === undefined ? {} : { relative: options.relative },
|
||||
})
|
||||
},
|
||||
async drag(from, to, options) {
|
||||
return action('drag', { ...coordinateParams(appHandle, windowId),
|
||||
return base.action('drag', { ...coordinateParams(appHandle, windowId),
|
||||
fromX: from[0], fromY: from[1], toX: to[0], toY: to[1],
|
||||
...options?.relative === undefined ? {} : { relative: options.relative },
|
||||
...options?.mouseButton === undefined ? {} : { mouseButton: options.mouseButton },
|
||||
@@ -223,26 +257,30 @@ function createCuaObjects(dispatch: CuaDispatch, sink: CuaObservationSink | unde
|
||||
},
|
||||
}
|
||||
}
|
||||
function createApp(appHandle: CuaAppHandle): CuaApp {
|
||||
function createCuaApp(appHandle: CuaAppHandle, capabilities: CuaCapabilities): CuaApp {
|
||||
const base = createCuaSurface({ appHandle }, params => ({ ...params, appHandle }), () => {
|
||||
if (!capabilities.appLevelAccessibility) {
|
||||
throw new Error('Application-level accessibility is not supported on current platform; observe a window instead.')
|
||||
}
|
||||
}, () => {
|
||||
if (!capabilities.keyboard.includes('app')) {
|
||||
throw new Error('App-scoped keyboard input is not supported on current platform.')
|
||||
}
|
||||
})
|
||||
return {
|
||||
...createSurface(appHandle), appHandle,
|
||||
async activate() { return action('activate', { appHandle }) },
|
||||
...base.surface, appHandle,
|
||||
async activate() { return base.action('activate', {}) },
|
||||
async getWindows(options) {
|
||||
const windows = WindowsResult.parse(await call('listWindows', { appHandle }))
|
||||
const windows = WindowsResult.parse(await base.call('listWindows', {}))
|
||||
await emit(sink, windows, options)
|
||||
return windows
|
||||
},
|
||||
async getWindow(windowId): Promise<CuaWindow> {
|
||||
const window = createWindow(appHandle, windowId)
|
||||
const window = createCuaWindow(appHandle, windowId, capabilities)
|
||||
await window.getAccessibilityTree({ emit: false })
|
||||
return window
|
||||
},
|
||||
async typeText(text) { return action('typeText', { appHandle, text }) },
|
||||
async paste(text, options) { return action('paste', { appHandle, text,
|
||||
...options?.format === undefined ? {} : { format: options.format },
|
||||
}) },
|
||||
async pressKey(key) { return action('pressKey', { appHandle, ...keyChord(key) }) },
|
||||
}
|
||||
}
|
||||
return { createApp, createWindow }
|
||||
return { createCuaApp }
|
||||
}
|
||||
@@ -16,7 +16,7 @@ import { NativeClient, NativeError, type NativeClientConfig, type NativeChannelF
|
||||
import { CuaPermissionStore } from './permission-store.ts'
|
||||
import { ApplicationAuthorizationResult, installCuaAccess } from './access.ts'
|
||||
import type { CuaApplicationGrant, CuaApplicationId } from './access-types.ts'
|
||||
import type { CuaAppHandle, CuaOperation } from './types.ts'
|
||||
import type { CuaAppHandle, CuaOperation, CuaStartupCapabilities } from './types.ts'
|
||||
import { ApplicationIcon, AppResult, NativeScreenshotResult, Request, ScreenshotResult, StateResult, WindowsResult } from './protocol.ts'
|
||||
import { CuaCoordinates } from './coordinates.ts'
|
||||
import { createScope, type Scope } from '@deepseek-ai/dsh-scope'
|
||||
@@ -29,6 +29,7 @@ import type {} from '@deepseek-ai/dsh-ptc-runtime'
|
||||
import type {} from '@deepseek-ai/cordis-plugin-loader'
|
||||
|
||||
export { CuaAccess } from './access.ts'
|
||||
export type { CuaStartupCapabilities } from './types.ts'
|
||||
|
||||
/** Native executable and per-operation resource bounds. */
|
||||
export interface CuaConfig {
|
||||
@@ -145,11 +146,12 @@ async function screenshotStore(ctx: Context, execution: Readonly<CuaExecution>)
|
||||
return { attachments, target: (image: ImageAttachmentRef) => llm.imageRequestTarget(provider, model, image) }
|
||||
}
|
||||
|
||||
/** Prepared platform executable, transport, and platform-specific operating instructions. */
|
||||
/** Prepared platform executable, transport, platform-specific instructions and startup capabilities. */
|
||||
export interface CuaBackend {
|
||||
helperPath: string
|
||||
createChannel: NativeChannelFactory
|
||||
guidance: string
|
||||
capabilities: CuaStartupCapabilities
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -282,7 +284,19 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend)
|
||||
return selected
|
||||
}
|
||||
signal.throwIfAborted()
|
||||
if (method === 'getCapabilities') return { ...z.record(z.string(), z.json()).parse(result), coordinateSpace: 'screenshot-pixels' }
|
||||
if (method === 'getCapabilities') {
|
||||
const runtime = z.record(z.string(), z.json()).parse(result)
|
||||
const capabilities = backend.capabilities
|
||||
return {
|
||||
...runtime,
|
||||
platform: capabilities.platform,
|
||||
methods: [...capabilities.methods],
|
||||
appLevelAccessibility: capabilities.appLevelAccessibility,
|
||||
keyboard: [...capabilities.keyboard],
|
||||
browser: capabilities.browser,
|
||||
coordinateSpace: capabilities.coordinateSpace,
|
||||
}
|
||||
}
|
||||
if (method === 'getState') {
|
||||
const state = StateResult.parse(result)
|
||||
execution.coordinates.observe(state.appHandle, state.windows, state.windowId ?? undefined)
|
||||
|
||||
@@ -13,16 +13,14 @@ function variants(schema: Schema): Schema[] { return schema.anyOf ?? schema.oneO
|
||||
|
||||
/**
|
||||
* Derive native request schemas and admission fields from the shared operation definitions.
|
||||
* @returns Model operation names, native field and enum tables, and the complete wire schema.
|
||||
* @returns Native field and enum tables and the complete wire schema.
|
||||
*/
|
||||
export function nativeProtocol(): {
|
||||
fields: Map<string, Set<string>>
|
||||
enums: Map<string, Map<string, Set<string>>>
|
||||
modelMethods: string[]
|
||||
json: string
|
||||
} {
|
||||
const resources = new Map(Object.entries(NativeResourceParameters))
|
||||
const modelMethods = Request.options.map(schema => schema.shape.method.value).sort()
|
||||
const schemas = [...Request.options, ...NativeOnlyOperations]
|
||||
.map(schema => z.toJSONSchema(schema))
|
||||
const fields = new Map<string, Set<string>>()
|
||||
@@ -69,7 +67,7 @@ export function nativeProtocol(): {
|
||||
fields.set(method, names)
|
||||
if (values.size) enums.set(method, values)
|
||||
}
|
||||
return { fields, enums, modelMethods, json: JSON.stringify({
|
||||
return { fields, enums, json: JSON.stringify({
|
||||
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
||||
$comment: 'Generated from the shared CUA operation schema. Do not edit.',
|
||||
title: 'dsh-cua native request',
|
||||
|
||||
@@ -17,7 +17,7 @@ const window = { ...app, windowId: id }
|
||||
const jsonValue = z.json()
|
||||
export const elementId = z.string().min(1)
|
||||
.describe('Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot.')
|
||||
const snapshot = { ...app, snapshotId: z.string().min(1), elementId }
|
||||
const snapshot = { ...app, windowId: id.optional(), snapshotId: z.string().min(1), elementId }
|
||||
const coordinate = { ...window, snapshotId: z.string().min(1).optional(), x: point, y: point }
|
||||
const absoluteCoordinate = { ...coordinate, relative: z.literal(false).optional().describe('Omitted or false uses pixels in the latest screenshot returned by this tool.') }
|
||||
const relativeCoordinate = { ...coordinate, relative: z.literal(true).describe('Use integer thousandths of the window width and height, from 0 to 999.'), x: relativePoint, y: relativePoint }
|
||||
@@ -59,9 +59,9 @@ export const Request = z.discriminatedUnion('method', [
|
||||
operation('setValue', { ...snapshot, value: jsonValue }),
|
||||
operation('selectText', { ...snapshot, text: z.string().min(1), prefix: z.string().optional(), suffix: z.string().optional(),
|
||||
selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }),
|
||||
operation('typeText', { ...app, text: z.string() }),
|
||||
operation('paste', { ...app, text: z.string(), format: z.enum(['text', 'html']).optional() }),
|
||||
operation('pressKey', { ...app, key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }),
|
||||
operation('typeText', { ...app, windowId: id.optional(), text: z.string() }),
|
||||
operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }),
|
||||
operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }),
|
||||
z.object({ method: z.literal('scroll'), params: z.union([
|
||||
z.object({ ...absoluteCoordinate,
|
||||
deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(),
|
||||
|
||||
@@ -45,6 +45,11 @@ export interface CuaAxOptions extends CuaObservationOptions {
|
||||
}
|
||||
/** Reported permissions and operations; null sessionLocked means no usable console session was verified. */
|
||||
export type CuaCapabilities = z.output<typeof CapabilitiesResult>
|
||||
/** Provider capabilities fixed when the provider is installed; runtime fields come from getCapabilities. */
|
||||
export type CuaStartupCapabilities = Pick<
|
||||
CuaCapabilities,
|
||||
'platform' | 'methods' | 'appLevelAccessibility' | 'keyboard' | 'browser' | 'coordinateSpace'
|
||||
>
|
||||
/** App discovery result accepted by getApp. */
|
||||
export type CuaAppInfo = Pick<Extract<CuaFoundAppInfo, { isRunning: true }>, 'id' | 'displayName' | 'isRunning' | 'pid'>
|
||||
/** An installed application match; running matches include a process identifier. */
|
||||
|
||||
@@ -19,6 +19,10 @@ const provider = {
|
||||
helperPath: resolve('unused-cua-helper'),
|
||||
createChannel() { throw new Error('Scope tests must not start a native helper') },
|
||||
guidance: 'Fixture CUA guidance',
|
||||
capabilities: {
|
||||
platform: 'macos', methods: ['getCapabilities', 'getApp', 'getState'], appLevelAccessibility: true,
|
||||
keyboard: ['app'], browser: false, coordinateSpace: 'screenshot-pixels',
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import { describe, expect, expectTypeOf, it, vi } from 'vitest'
|
||||
import { brandString } from '@deepseek-ai/dsh-brand'
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import { createCuaApi } from '../src/api.ts'
|
||||
import type { CuaApp, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts'
|
||||
import type { CuaApp, CuaCapabilities, CuaElementId, CuaOperation, CuaScreenshot, CuaWindow } from '../src/types.ts'
|
||||
|
||||
/** Element ids are opaque strings; these fixtures use the macOS decimal form. */
|
||||
const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
|
||||
@@ -13,7 +13,7 @@ const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG
|
||||
const bounds = { x: -640, y: 100, width: 640, height: 480 }
|
||||
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', bundleId: 'test.editor', running: true }
|
||||
const delivered = { dispatched: true }
|
||||
const capabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
|
||||
const capabilities: CuaCapabilities = { platform: 'macos', sessionLocked: false, permissions: { accessibility: true, screenRecording: false }, appLevelAccessibility: true, keyboard: ['app'], methods: ['getState', 'click'], coordinateSpace: 'screenshot-pixels', browser: false }
|
||||
|
||||
function windowInfo(windowId: number) {
|
||||
return { windowId, appName: 'Editor', title: `Document ${windowId}`, bounds, isOnScreen: true, onCurrentSpace: null }
|
||||
@@ -21,11 +21,12 @@ function windowInfo(windowId: number) {
|
||||
|
||||
function fixture(withSink = true) {
|
||||
let windows = [windowInfo(23)]
|
||||
let currentCapabilities: CuaCapabilities = capabilities
|
||||
let generation = 0
|
||||
const sink = { text: vi.fn<(value: string) => void>(), image: vi.fn<(screenshot: CuaScreenshot) => void>() }
|
||||
const dispatch = vi.fn(async ({ method, params }: CuaOperation): Promise<JsonValue> => {
|
||||
switch (method) {
|
||||
case 'getCapabilities': return capabilities
|
||||
case 'getCapabilities': return currentCapabilities
|
||||
case 'listApps': return [selected, { pid: 18, name: 'Utility' }]
|
||||
case 'getApp': return selected
|
||||
case 'listWindows': return windows
|
||||
@@ -45,6 +46,7 @@ function fixture(withSink = true) {
|
||||
dispatch, sink, api: createCuaApi(dispatch, withSink ? sink : undefined),
|
||||
setWindows: (ids: number[]) => { windows = ids.map(windowInfo) },
|
||||
setWindowRecords: (records: ReturnType<typeof windowInfo>[]) => { windows = records },
|
||||
setCapabilities: (value: CuaCapabilities) => { currentCapabilities = value },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,7 +165,7 @@ describe('hover across observations', () => {
|
||||
await window.getScreenshot({ emit: false })
|
||||
expect(await window.hover([30, 40])).toEqual({ ...delivered, hoverRoute: 'accessibility', systemPointerMoved: false })
|
||||
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0' } },
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: 23 } },
|
||||
{ method: 'getState', params: { appHandle: 'app-17', windowId: 23 } },
|
||||
{ method: 'screenshot', params: { appHandle: 'app-17', windowId: 23 } },
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', windowId: 23, x: 30, y: 40 } },
|
||||
@@ -199,7 +201,7 @@ describe('pointer buttons and modifiers', () => {
|
||||
const window = await app.getWindow(23)
|
||||
await window.click(element('0'), { mouseButton: 'middle', modifiers: ['shift'] })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', mouseButton: 'middle', modifiers: ['shift'],
|
||||
appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '0', windowId: 23, mouseButton: 'middle', modifiers: ['shift'],
|
||||
} })
|
||||
await window.click([10, 20], { modifiers: ['command'] })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: {
|
||||
@@ -322,18 +324,18 @@ describe('application and window observations', () => {
|
||||
await second.click(element('3'))
|
||||
expect(f.dispatch.mock.calls.slice(-3).map(([operation]) => operation)).toEqual([
|
||||
{ method: 'setValue', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '1', value: 'First document' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '2', windowId: 23 } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-3', elementId: '3', windowId: 24 } },
|
||||
])
|
||||
expect(first.windowId).toBe(23)
|
||||
expect(second.windowId).toBe(24)
|
||||
expect(f.sink.text.mock.calls.map(([text]) => text)).toEqual([])
|
||||
await expect(second.getAccessibilityTree({ emit: false, disableDiffing: true })).resolves.toBe('Tree 4')
|
||||
await second.click(element('3'))
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3' } })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-4', elementId: '3', windowId: 24 } })
|
||||
await app.getAccessibilityTree({ emit: false })
|
||||
await first.invoke(element('4'))
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4' } })
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: { appHandle: 'app-17', snapshotId: 'snapshot-5', elementId: '4', windowId: 23 } })
|
||||
expect(f.sink.text).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -476,6 +478,69 @@ describe('application and window observations', () => {
|
||||
await expect(app.getWindows()).resolves.toEqual([observed])
|
||||
})
|
||||
|
||||
it('skips app-level accessibility while keeping window observation available', async () => {
|
||||
const f = fixture()
|
||||
f.setCapabilities({ ...capabilities, appLevelAccessibility: false })
|
||||
const app = await f.api.getApp('Editor')
|
||||
expect(f.dispatch.mock.calls.map(([call]) => call.method)).toEqual(['getCapabilities', 'getApp'])
|
||||
await expect(app.getAccessibilityTree()).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
|
||||
await expect(app.listActions(element('0'))).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
|
||||
await expect(app.performAction(element('0'), 'AXPress')).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
|
||||
await expect(app.invoke(element('0'))).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
|
||||
await expect(app.setValue(element('0'), 'value')).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
|
||||
await expect(app.selectText(element('0'), 'text')).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(2)
|
||||
f.dispatch.mockClear()
|
||||
await expect(app.typeText('keyboard without accessibility')).resolves.toEqual(delivered)
|
||||
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
|
||||
{ method: 'typeText', params: { appHandle: 'app-17', text: 'keyboard without accessibility' } },
|
||||
])
|
||||
const window = await app.getWindow(23)
|
||||
await expect(window.getAccessibilityTree({ emit: false })).resolves.toBe('Tree 2')
|
||||
await expect(app.getAccessibilityTree()).rejects.toThrow('Application-level accessibility is not supported on current platform; observe a window instead.')
|
||||
await expect(window.invoke(element('0'))).resolves.toEqual(delivered)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'invoke', params: {
|
||||
appHandle: 'app-17', windowId: 23, snapshotId: 'snapshot-2', elementId: '0',
|
||||
} })
|
||||
})
|
||||
|
||||
it('honors window keyboard support and keeps app keyboard available', async () => {
|
||||
const f = fixture()
|
||||
f.setCapabilities({ ...capabilities, keyboard: ['app', 'window'] })
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
f.dispatch.mockClear()
|
||||
await window.typeText('hello')
|
||||
await window.paste('<b>bold</b>', { format: 'html' })
|
||||
await window.pressKey('cmd+a')
|
||||
await app.typeText('app text')
|
||||
await app.paste('app paste')
|
||||
await app.pressKey('shift+Tab')
|
||||
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
|
||||
{ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'hello' } },
|
||||
{ method: 'paste', params: { appHandle: 'app-17', windowId: 23, text: '<b>bold</b>', format: 'html' } },
|
||||
{ method: 'pressKey', params: { appHandle: 'app-17', windowId: 23, key: 'a', modifiers: ['command'] } },
|
||||
{ method: 'typeText', params: { appHandle: 'app-17', text: 'app text' } },
|
||||
{ method: 'paste', params: { appHandle: 'app-17', text: 'app paste' } },
|
||||
{ method: 'pressKey', params: { appHandle: 'app-17', key: 'Tab', modifiers: ['shift'] } },
|
||||
])
|
||||
})
|
||||
|
||||
it('rejects app keyboard when only window scope is supported', async () => {
|
||||
const f = fixture()
|
||||
f.setCapabilities({ ...capabilities, keyboard: ['window'] })
|
||||
const app = await f.api.getApp('Editor')
|
||||
const window = await app.getWindow(23)
|
||||
f.dispatch.mockClear()
|
||||
await expect(app.typeText('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
await expect(app.paste('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
await expect(app.pressKey('Return')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
await expect(app.pressKey('unknown+a')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
expect(f.dispatch).not.toHaveBeenCalled()
|
||||
await expect(window.typeText('window text')).resolves.toEqual(delivered)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'window text' } })
|
||||
})
|
||||
|
||||
it('accepts coordinate input without a current snapshot', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
@@ -686,8 +751,9 @@ describe('input actions', () => {
|
||||
() => window.typeText('hello'),
|
||||
() => window.paste('hello'),
|
||||
() => window.pressKey('Return'),
|
||||
() => window.pressKey('unknown+a'),
|
||||
]) {
|
||||
await expect(run()).rejects.toThrow('Keyboard input is not supported at this scope.')
|
||||
await expect(run()).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.')
|
||||
}
|
||||
expect(f.dispatch).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -699,8 +765,8 @@ describe('input actions', () => {
|
||||
await window.click(element('12'), { mouseButton: 'right' })
|
||||
await window.click(element('13'), { clickCount: 2 })
|
||||
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', mouseButton: 'right' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', clickCount: 2 } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '12', windowId: 23, mouseButton: 'right' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '13', windowId: 23, clickCount: 2 } },
|
||||
])
|
||||
})
|
||||
|
||||
@@ -711,15 +777,15 @@ describe('input actions', () => {
|
||||
await window.click(element('42.12345'))
|
||||
await window.hover(element('42.12345'))
|
||||
expect(f.dispatch.mock.calls.slice(-2).map(([operation]) => operation)).toEqual([
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } },
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345' } },
|
||||
{ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: 23 } },
|
||||
{ method: 'hover', params: { appHandle: 'app-17', snapshotId: 'snapshot-2', elementId: '42.12345', windowId: 23 } },
|
||||
])
|
||||
})
|
||||
|
||||
it('does not activate on selection and dispatches only an explicit activation request', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
expect(f.dispatch.mock.calls.map(([call]) => call.method)).toEqual(['getApp', 'getState'])
|
||||
expect(f.dispatch.mock.calls.map(([call]) => call.method)).toEqual(['getCapabilities', 'getApp', 'getState'])
|
||||
await app.activate()
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'activate', params: { appHandle: 'app-17' } })
|
||||
})
|
||||
@@ -731,10 +797,10 @@ describe('input actions', () => {
|
||||
const failure = new Error('Application was restarted; select it again')
|
||||
f.dispatch.mockRejectedValueOnce(failure)
|
||||
await expect(app.typeText('hello')).rejects.toBe(failure)
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(4)
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(5)
|
||||
f.dispatch.mockResolvedValueOnce({ dispatched: 'yes' })
|
||||
await expect(window.click(element('0'))).rejects.toThrow()
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(5)
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(6)
|
||||
})
|
||||
|
||||
it('rejects invalid selection inputs before dispatch', async () => {
|
||||
|
||||
@@ -5,12 +5,17 @@ import { brandString } from '@deepseek-ai/dsh-brand'
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import { connectCua } from '../src/bridge.ts'
|
||||
import type { CuaToolBinding } from '../src/bridge.ts'
|
||||
import type { CuaElementId, CuaOperation } from '../src/types.ts'
|
||||
import type { CuaCapabilities, CuaElementId, CuaOperation } from '../src/types.ts'
|
||||
|
||||
/** Element ids are opaque strings; these fixtures use the macOS decimal form. */
|
||||
const element = (id: string): CuaElementId => brandString<CuaElementId>(id)
|
||||
|
||||
const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC'
|
||||
const capabilities: CuaCapabilities = {
|
||||
platform: 'macos', permissions: { accessibility: true, screenRecording: true }, appLevelAccessibility: true,
|
||||
keyboard: ['app'], sessionLocked: false, methods: ['getCapabilities', 'getApp', 'getState', 'screenshot'],
|
||||
coordinateSpace: 'screenshot-pixels', browser: false,
|
||||
}
|
||||
const selected = { appHandle: 'app-17', pid: 17, name: 'Editor', running: true }
|
||||
const bounds = { x: 0, y: 0, width: 640, height: 480 }
|
||||
const snapshot = {
|
||||
@@ -20,6 +25,7 @@ const snapshot = {
|
||||
|
||||
it('uses the current tool binding when a retained app is called by another execution', async () => {
|
||||
const first = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => {
|
||||
if (operation.method === 'getCapabilities') return capabilities
|
||||
if (operation.method === 'getApp') return selected
|
||||
if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null }
|
||||
return snapshot
|
||||
@@ -33,8 +39,8 @@ it('uses the current tool binding when a retained app is called by another execu
|
||||
current = second
|
||||
|
||||
await expect(window.click(element('0'))).resolves.toEqual({ dispatched: true })
|
||||
expect(first).toHaveBeenCalledTimes(3)
|
||||
expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0' } })
|
||||
expect(first).toHaveBeenCalledTimes(4)
|
||||
expect(second).toHaveBeenCalledWith({ method: 'click', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '0', windowId: 23 } })
|
||||
expect(text).not.toHaveBeenCalled()
|
||||
current = first
|
||||
await app.getAccessibilityTree()
|
||||
@@ -61,6 +67,7 @@ it.each([null, 'text only', { content: [{ type: 'text', text: '{}' }] }])('rejec
|
||||
|
||||
it('returns screenshot bytes and metadata with one image-bearing tool call and no image text output', async () => {
|
||||
const tool = vi.fn(async (operation: CuaOperation): Promise<JsonValue> => {
|
||||
if (operation.method === 'getCapabilities') return capabilities
|
||||
if (operation.method === 'getApp') return selected
|
||||
if (operation.method === 'getState') return { ...snapshot, windowId: operation.params.windowId ?? null }
|
||||
return {
|
||||
@@ -73,6 +80,6 @@ it('returns screenshot bytes and metadata with one image-bearing tool call and n
|
||||
text.mockClear()
|
||||
|
||||
expect(await window.getScreenshot()).toEqual({ bytes: Uint8Array.from(Buffer.from(PNG, 'base64')), mimeType: 'image/png', pixelWidth: 1, pixelHeight: 1, windowId: 23 })
|
||||
expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getApp', 'getState', 'getState', 'screenshot'])
|
||||
expect(tool.mock.calls.map(([operation]) => operation.method)).toEqual(['getCapabilities', 'getApp', 'getState', 'getState', 'screenshot'])
|
||||
expect(text).not.toHaveBeenCalled()
|
||||
})
|
||||
@@ -33,7 +33,7 @@ it('retains optional resource fields and tuple-valued parameters without inventi
|
||||
})
|
||||
|
||||
it('admits only locator fields for default invocation', () => {
|
||||
expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId']))
|
||||
expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId', 'windowId']))
|
||||
})
|
||||
|
||||
it('accepts a parameter-free object schema without a properties dictionary', () => {
|
||||
|
||||
Reference in new issue
Block a user