feat(cua): separate key modifiers and add directional key events

This commit is contained in:
Huo Yaoyuan committed 2026-11-05 15:05:52 +08:00
1 parent 7fc481337d
commit 2dfe4912dd
9 files changed
+192 -60

No files matched your search

@@ -189,6 +189,9 @@ actor AXWorker {
trace: ScreenshotTrace? = nil) async throws -> DriverOutcome {
try cancellation.check()
guard !closed else { throw NativeFailure(code: "NATIVE_CLOSED", message: "Helper is closing") }
if method == "sendKeyEvent" {
throw NativeFailure(code: "NOT_IMPLEMENTED", message: "sendKeyEvent is not implemented on current platform.")
}
let locked = observeSession()
if method == "getCapabilities" {
return .result(.object(["permissions": .object([
@@ -24,6 +24,7 @@ enum NativeProtocol {
"screenshot": ["appHandle", "windowId"],
"scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"],
"selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text", "windowId"],
"sendKeyEvent": ["appHandle", "direction", "key", "windowId"],
"setValue": ["appHandle", "elementId", "snapshotId", "value", "windowId"],
"show": ["appHandle", "windowId"],
"typeText": ["appHandle", "text", "windowId"],
@@ -33,8 +34,8 @@ enum NativeProtocol {
"drag": ["modifiers": ["command", "control", "fn", "option", "shift"], "mouseButton": ["left", "middle", "right"]],
"openPermissionSettings": ["permission": ["accessibility", "screenRecording"]],
"paste": ["format": ["html", "text"]],
"pressKey": ["modifiers": ["command", "control", "fn", "option", "shift"]],
"requestPermission": ["permission": ["accessibility", "screenRecording"]],
"selectText": ["selectionType": ["cursor_after", "cursor_before", "text"]],
"sendKeyEvent": ["direction": ["down", "up"]],
]
}
@@ -991,14 +991,7 @@
"modifiers": {
"type": "array",
"items": {
"type": "string",
"enum": [
"command",
"control",
"option",
"shift",
"fn"
]
"type": "string"
}
}
},
@@ -1016,6 +1009,55 @@
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"id": {
"type": "string"
},
"method": {
"type": "string",
"const": "sendKeyEvent"
},
"params": {
"type": "object",
"properties": {
"appHandle": {
"type": "string",
"minLength": 1
},
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 4294967295
},
"key": {
"type": "string",
"minLength": 1
},
"direction": {
"type": "string",
"enum": [
"down",
"up"
]
}
},
"required": [
"appHandle",
"key",
"direction"
],
"additionalProperties": false
}
},
"required": [
"id",
"method",
"params"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
+6 -27
View File
@@ -23,31 +23,6 @@ function requestIssueMessage(issues: readonly z.core.$ZodIssue[], parent: readon
}))].join('; ')
}
/** Parse a portable key chord.
* @param chord Key or chord.
* @returns Native key and modifier names.
*/
export function keyChord(chord: string): { key: string; modifiers: string[] } {
const parts = chord.split('+')
let key = parts.pop()
if (key === '' && parts.at(-1) === '') {
key = '+'
parts.pop()
}
if (!key) throw new Error('A key chord must end with a key name')
const names: Record<string, string> = {
cmd: 'command', command: 'command', super: 'command', Super_L: 'command', Super_R: 'command',
ctrl: 'control', control: 'control', Control_L: 'control', Control_R: 'control',
alt: 'option', option: 'option', Alt_L: 'option', Alt_R: 'option',
shift: 'shift', Shift_L: 'shift', Shift_R: 'shift', fn: 'fn',
}
return { key, modifiers: parts.map((part) => {
const modifier = names[part]
if (modifier === undefined) throw new Error(`Unsupported key modifier: ${part}`)
return modifier
}) }
}
function validatedCall(dispatch: CuaDispatch) {
return (method: CuaOperation['method'], params: Record<string, JsonValue>) => {
const request = Request.safeParse({ method, params })
@@ -188,9 +163,13 @@ function createCuaObjects(call: ReturnType<typeof validatedCall>, sink: CuaObser
...options?.format === undefined ? {} : { format: options.format },
})
},
async pressKey(key) {
async pressKey(key, modifiers = []) {
requireKeyboard()
return action('pressKey', keyChord(key))
return action('pressKey', { key, modifiers: [...modifiers] })
},
async sendKeyEvent(key, direction) {
requireKeyboard()
return action('sendKeyEvent', { key, direction })
},
}
return { surface, call: scopedCall, action, elementParams }
+2 -1
View File
@@ -61,7 +61,8 @@ export const Request = z.discriminatedUnion('method', [
selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }),
operation('typeText', { ...app, windowId: id.optional(), text: z.string() }),
operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }),
operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }),
operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.string()).optional() }),
operation('sendKeyEvent', { ...app, windowId: id.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }),
z.object({ method: z.literal('scroll'), params: z.union([
z.object({ ...absoluteCoordinate,
deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(),
+13 -3
View File
@@ -160,12 +160,22 @@ export interface CuaSurface {
*/
paste(text: string, options?: CuaPasteOptions): Promise<CuaActionResult>
/**
* Send a key or chord at the current input focus.
* Press and release one key at the current input focus.
* Rejects when the provider does not support keyboard input at this scope.
* @param key - xdotool-style chord such as super+c, shift+Tab, + or cmd++; plus is also a key name.
* @param key - Provider-defined key name to press and release; not a chord.
* @param modifiers - Platform- or provider-defined modifier names held for this keystroke; omitted means none.
* @returns Whether the provider successfully executed the requested operation.
*/
pressKey(key: string): Promise<CuaActionResult>
pressKey(key: string, modifiers?: string[]): Promise<CuaActionResult>
/**
* Send one key-down or key-up event at the current input focus.
* Does not send the opposite event.
* Rejects when the provider does not support keyboard input at this scope.
* @param key - Platform-defined key name; modifier keys may themselves be targets.
* @param direction - Explicit event direction; the opposite event is never sent implicitly.
* @returns Whether the provider successfully executed the requested operation.
*/
sendKeyEvent(key: string, direction: 'down' | 'up'): Promise<CuaActionResult>
}
/**
* An explicit window with screenshot and pointer input operations.
+87 -19
View File
@@ -514,10 +514,10 @@ describe('application and window observations', () => {
f.dispatch.mockClear()
await window.typeText('hello')
await window.paste('<b>bold</b>', { format: 'html' })
await window.pressKey('cmd+a')
await window.pressKey('a', ['command'])
await app.typeText('app text')
await app.paste('app paste')
await app.pressKey('shift+Tab')
await app.pressKey('Tab', ['shift'])
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'hello' } },
{ method: 'paste', params: { appHandle: 'app-17', windowId: 23, text: '<b>bold</b>', format: 'html' } },
@@ -537,7 +537,7 @@ describe('application and window observations', () => {
await expect(app.typeText('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
await expect(app.paste('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
await expect(app.pressKey('Return')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
await expect(app.pressKey('unknown+a')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
await expect(app.pressKey('', ['command'])).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
expect(f.dispatch).not.toHaveBeenCalled()
await expect(window.typeText('window text')).resolves.toEqual(delivered)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'window text' } })
@@ -684,11 +684,7 @@ describe('input actions', () => {
['oversized scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })],
['fractional scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })],
['negative drag point', ({ window }) => window.drag([0, 0], [-1, 3])],
['empty chord', ({ app }) => app.pressKey('')],
['missing key', ({ app }) => app.pressKey('super+')],
['empty plus modifier', ({ app }) => app.pressKey('++')],
['extra plus delimiter', ({ app }) => app.pressKey('cmd+++')],
['unknown modifier', ({ app }) => app.pressKey('unknown+a')],
['empty key', ({ app }) => app.pressKey('')],
]
it.each<[string, (surfaces: Surfaces) => Promise<unknown>]>([
['getState windowId: Too small: expected number to be >0', ({ app }) => app.getWindow(0)],
@@ -714,17 +710,41 @@ describe('input actions', () => {
expect(f.dispatch).toHaveBeenCalledTimes(calls)
})
it.each<[string, string, string[]]>([
['+', '+', []],
['cmd++', '+', ['command']],
['ctrl+shift++', '+', ['control', 'shift']],
['cmd+plus', 'plus', ['command']],
['plus', 'plus', []],
])('dispatches the plus-key chord %s', async (chord, key, modifiers) => {
it.each<[string, string[] | undefined, string[]]>([
['+', undefined, []],
['plus', [], []],
['+', ['command'], ['command']],
['Tab', ['shift'], ['shift']],
])('forwards literal key %s with modifiers %j without splitting', async (key, modifiers, expected) => {
const f = fixture()
const app = await f.api.getApp('Editor')
await app.pressKey(chord)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { appHandle: 'app-17', key, modifiers } })
if (modifiers === undefined) await app.pressKey(key)
else await app.pressKey(key, modifiers)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { appHandle: 'app-17', key, modifiers: expected } })
})
it('forwards the literal key unsplit in one RPC without mutating modifiers', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
const modifiers: string[] = ['control', 'shift']
f.dispatch.mockClear()
await app.pressKey('+', modifiers)
expect(f.dispatch).toHaveBeenCalledTimes(1)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: {
appHandle: 'app-17', key: '+', modifiers: ['control', 'shift'],
} })
expect(modifiers).toEqual(['control', 'shift'])
})
it('forwards arbitrary provider-defined key and modifier names in one RPC', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockClear()
await app.pressKey('PlatformKey', ['platform-modifier'])
expect(f.dispatch).toHaveBeenCalledTimes(1)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: {
appHandle: 'app-17', key: 'PlatformKey', modifiers: ['platform-modifier'],
} })
})
it('passes literal text and keys to the selected app identity', async () => {
@@ -733,7 +753,7 @@ describe('input actions', () => {
await app.typeText('中文\nline two')
await app.paste('粘贴\nplain text')
await app.paste('<b>bold</b>', { format: 'html' })
await app.pressKey('Control_L+shift+a')
await app.pressKey('a', ['control', 'shift'])
await app.pressKey('Return')
expect(f.dispatch.mock.calls.slice(-5).map(([operation]) => operation)).toEqual([
{ method: 'typeText', params: { appHandle: 'app-17', text: '中文\nline two' } },
@@ -753,13 +773,61 @@ describe('input actions', () => {
() => window.typeText('hello'),
() => window.paste('hello'),
() => window.pressKey('Return'),
() => window.pressKey('unknown+a'),
() => window.pressKey('', ['command']),
]) {
await expect(run()).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.')
}
expect(f.dispatch).not.toHaveBeenCalled()
})
it.each<[string, 'down' | 'up']>([
['app', 'down'],
['app', 'up'],
['window', 'down'],
['window', 'up'],
])('sends one explicit %s-scoped key event (%s) with exactly one RPC', async (scope, direction) => {
const f = fixture()
f.setCapabilities({ ...capabilities, keyboard: ['app', 'window'] })
const app = await f.api.getApp('Editor')
const surface = scope === 'window' ? await app.getWindow(23) : app
f.dispatch.mockClear()
f.dispatch.mockResolvedValueOnce(delivered)
await expect(surface.sendKeyEvent('Shift', direction)).resolves.toEqual(delivered)
expect(f.dispatch).toHaveBeenCalledTimes(1)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'sendKeyEvent', params: {
appHandle: 'app-17', ...scope === 'window' ? { windowId: 23 } : {}, key: 'Shift', direction,
} })
})
it('returns an un-dispatched key-event result from one RPC without retrying', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockClear()
f.dispatch.mockResolvedValueOnce({ dispatched: false })
await expect(app.sendKeyEvent('Command', 'down')).resolves.toEqual({ dispatched: false })
expect(f.dispatch).toHaveBeenCalledTimes(1)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'sendKeyEvent', params: {
appHandle: 'app-17', key: 'Command', direction: 'down',
} })
})
it('rejects key events without keyboard support before validating them', async () => {
const appFixture = fixture()
appFixture.setCapabilities({ ...capabilities, keyboard: ['window'] })
const app = await appFixture.api.getApp('Editor')
appFixture.dispatch.mockClear()
await expect(app.sendKeyEvent('', 'down')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
expect(appFixture.dispatch).not.toHaveBeenCalled()
const windowFixture = fixture()
windowFixture.setCapabilities({ ...capabilities, keyboard: ['app'] })
const windowApp = await windowFixture.api.getApp('Editor')
const window = await windowApp.getWindow(23)
windowFixture.dispatch.mockClear()
await expect(window.sendKeyEvent('', 'up')).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.')
expect(windowFixture.dispatch).not.toHaveBeenCalled()
})
it('retains the snapshot identity for element right and double clicks', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
@@ -36,6 +36,20 @@ it('admits only locator fields for default invocation', () => {
expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId', 'windowId']))
})
it('admits arbitrary pressKey modifier names without a generated string enum', () => {
const native = nativeProtocol()
expect(native.fields.get('pressKey')).toContain('modifiers')
expect(native.enums.get('pressKey')?.has('modifiers') ?? false).toBe(false)
})
it('admits key events with only an explicit direction enum', () => {
const native = nativeProtocol()
expect(native.fields.get('sendKeyEvent')).toEqual(new Set(['appHandle', 'windowId', 'key', 'direction']))
expect(native.enums.get('sendKeyEvent')?.get('direction')).toEqual(new Set(['down', 'up']))
expect(native.enums.get('sendKeyEvent')?.has('modifiers') ?? false).toBe(false)
expect(native.enums.get('sendKeyEvent')?.has('key') ?? false).toBe(false)
})
it('accepts a parameter-free object schema without a properties dictionary', () => {
replacements.values.set(0, { properties: { method: { const: 'getCapabilities' }, params: { type: 'object' } } })
expect(nativeProtocol().fields.get('getCapabilities')).toEqual(new Set())
@@ -12,13 +12,27 @@ const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [
{ reason: 'numeric element id', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: 0 } } },
{ reason: 'coordinate click without a window', request: { method: 'click', params: { appHandle: 'app', x: 1, y: 2 } } },
{ reason: 'mixed element and coordinate click', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', windowId: 23, x: 1, y: 2 } } },
{ reason: 'unsupported modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['hyper'] } } },
{ reason: 'non-string pressKey modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['command', 7] } } },
{ reason: 'sendKeyEvent modifiers', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a', direction: 'down', modifiers: [] } } },
{ reason: 'sendKeyEvent without direction', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a' } } },
{ reason: 'sendKeyEvent invalid direction', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a', direction: 'press' } } },
{ reason: 'sendKeyEvent empty key', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: '', direction: 'down' } } },
]
it.each(rejectedRequests)('rejects $reason at operation admission', ({ request }) => {
expect(Request.safeParse(request).success).toBe(false)
})
it('admits arbitrary provider-defined pressKey modifier names', () => {
const request = { method: 'pressKey', params: { appHandle: 'app', key: 'PlatformKey', modifiers: ['platform-modifier', 'shift'] } }
expect(Request.parse(request)).toEqual(request)
})
it('admits arbitrary key names with an explicit key-event direction', () => {
const request = { method: 'sendKeyEvent', params: { appHandle: 'app', windowId: 23, key: 'Command', direction: 'down' } }
expect(Request.parse(request)).toEqual(request)
})
it.each([
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0' },
{ appHandle: 'app', windowId: 23, x: 1, y: 2 },