From 2dfe4912dd193330ea192dec7d18ca8d2fc0a2e3 Mon Sep 17 00:00:00 2001 From: Huo Yaoyuan Date: Thu, 5 Nov 2026 15:05:52 +0800 Subject: [PATCH] feat(cua): separate key modifiers and add directional key events --- .../native/Sources/CuaNative/AXWorker.swift | 3 + .../CuaNative/NativeProtocol.generated.swift | 3 +- .../cua-macos/native/protocol.schema.json | 58 ++++++++-- packages/experimental/cua/src/api.ts | 33 +----- packages/experimental/cua/src/protocol.ts | 3 +- packages/experimental/cua/src/types.ts | 16 ++- packages/experimental/cua/tests/api.spec.ts | 106 ++++++++++++++---- .../cua/tests/native-schema.spec.ts | 14 +++ .../experimental/cua/tests/protocol.spec.ts | 16 ++- 9 files changed, 192 insertions(+), 60 deletions(-) diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index c1370a2ba5..01071ca6f4 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -189,6 +189,9 @@ actor AXWorker { trace: ScreenshotTrace? = nil) async throws -> DriverOutcome { try cancellation.check() guard !closed else { throw NativeFailure(code: "NATIVE_CLOSED", message: "Helper is closing") } + if method == "sendKeyEvent" { + throw NativeFailure(code: "NOT_IMPLEMENTED", message: "sendKeyEvent is not implemented on current platform.") + } let locked = observeSession() if method == "getCapabilities" { return .result(.object(["permissions": .object([ diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift index 9f26c8c2ab..09b2f203cf 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift @@ -24,6 +24,7 @@ enum NativeProtocol { "screenshot": ["appHandle", "windowId"], "scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"], "selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text", "windowId"], + "sendKeyEvent": ["appHandle", "direction", "key", "windowId"], "setValue": ["appHandle", "elementId", "snapshotId", "value", "windowId"], "show": ["appHandle", "windowId"], "typeText": ["appHandle", "text", "windowId"], @@ -33,8 +34,8 @@ enum NativeProtocol { "drag": ["modifiers": ["command", "control", "fn", "option", "shift"], "mouseButton": ["left", "middle", "right"]], "openPermissionSettings": ["permission": ["accessibility", "screenRecording"]], "paste": ["format": ["html", "text"]], - "pressKey": ["modifiers": ["command", "control", "fn", "option", "shift"]], "requestPermission": ["permission": ["accessibility", "screenRecording"]], "selectText": ["selectionType": ["cursor_after", "cursor_before", "text"]], + "sendKeyEvent": ["direction": ["down", "up"]], ] } diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index d6749c5951..b110cb2b3c 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -991,14 +991,7 @@ "modifiers": { "type": "array", "items": { - "type": "string", - "enum": [ - "command", - "control", - "option", - "shift", - "fn" - ] + "type": "string" } } }, @@ -1016,6 +1009,55 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "method": { + "type": "string", + "const": "sendKeyEvent" + }, + "params": { + "type": "object", + "properties": { + "appHandle": { + "type": "string", + "minLength": 1 + }, + "windowId": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 4294967295 + }, + "key": { + "type": "string", + "minLength": 1 + }, + "direction": { + "type": "string", + "enum": [ + "down", + "up" + ] + } + }, + "required": [ + "appHandle", + "key", + "direction" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "method", + "params" + ], + "additionalProperties": false + }, { "type": "object", "properties": { diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index 0da477e19f..692c0f1eae 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -23,31 +23,6 @@ function requestIssueMessage(issues: readonly z.core.$ZodIssue[], parent: readon }))].join('; ') } -/** Parse a portable key chord. - * @param chord Key or chord. - * @returns Native key and modifier names. - */ -export function keyChord(chord: string): { key: string; modifiers: string[] } { - const parts = chord.split('+') - let key = parts.pop() - if (key === '' && parts.at(-1) === '') { - key = '+' - parts.pop() - } - if (!key) throw new Error('A key chord must end with a key name') - const names: Record = { - cmd: 'command', command: 'command', super: 'command', Super_L: 'command', Super_R: 'command', - ctrl: 'control', control: 'control', Control_L: 'control', Control_R: 'control', - alt: 'option', option: 'option', Alt_L: 'option', Alt_R: 'option', - shift: 'shift', Shift_L: 'shift', Shift_R: 'shift', fn: 'fn', - } - return { key, modifiers: parts.map((part) => { - const modifier = names[part] - if (modifier === undefined) throw new Error(`Unsupported key modifier: ${part}`) - return modifier - }) } -} - function validatedCall(dispatch: CuaDispatch) { return (method: CuaOperation['method'], params: Record) => { const request = Request.safeParse({ method, params }) @@ -188,9 +163,13 @@ function createCuaObjects(call: ReturnType, sink: CuaObser ...options?.format === undefined ? {} : { format: options.format }, }) }, - async pressKey(key) { + async pressKey(key, modifiers = []) { requireKeyboard() - return action('pressKey', keyChord(key)) + return action('pressKey', { key, modifiers: [...modifiers] }) + }, + async sendKeyEvent(key, direction) { + requireKeyboard() + return action('sendKeyEvent', { key, direction }) }, } return { surface, call: scopedCall, action, elementParams } diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index e1e88399ee..5ec217f1f5 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -61,7 +61,8 @@ export const Request = z.discriminatedUnion('method', [ selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }), operation('typeText', { ...app, windowId: id.optional(), text: z.string() }), operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }), - operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }), + operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.string()).optional() }), + operation('sendKeyEvent', { ...app, windowId: id.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }), z.object({ method: z.literal('scroll'), params: z.union([ z.object({ ...absoluteCoordinate, deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(), diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 80ae7a4da1..c059a546f2 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -160,12 +160,22 @@ export interface CuaSurface { */ paste(text: string, options?: CuaPasteOptions): Promise /** - * Send a key or chord at the current input focus. + * Press and release one key at the current input focus. * Rejects when the provider does not support keyboard input at this scope. - * @param key - xdotool-style chord such as super+c, shift+Tab, + or cmd++; plus is also a key name. + * @param key - Provider-defined key name to press and release; not a chord. + * @param modifiers - Platform- or provider-defined modifier names held for this keystroke; omitted means none. * @returns Whether the provider successfully executed the requested operation. */ - pressKey(key: string): Promise + pressKey(key: string, modifiers?: string[]): Promise + /** + * Send one key-down or key-up event at the current input focus. + * Does not send the opposite event. + * Rejects when the provider does not support keyboard input at this scope. + * @param key - Platform-defined key name; modifier keys may themselves be targets. + * @param direction - Explicit event direction; the opposite event is never sent implicitly. + * @returns Whether the provider successfully executed the requested operation. + */ + sendKeyEvent(key: string, direction: 'down' | 'up'): Promise } /** * An explicit window with screenshot and pointer input operations. diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index a604eff2d5..3d6213b625 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -514,10 +514,10 @@ describe('application and window observations', () => { f.dispatch.mockClear() await window.typeText('hello') await window.paste('bold', { format: 'html' }) - await window.pressKey('cmd+a') + await window.pressKey('a', ['command']) await app.typeText('app text') await app.paste('app paste') - await app.pressKey('shift+Tab') + await app.pressKey('Tab', ['shift']) expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([ { method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'hello' } }, { method: 'paste', params: { appHandle: 'app-17', windowId: 23, text: 'bold', format: 'html' } }, @@ -537,7 +537,7 @@ describe('application and window observations', () => { await expect(app.typeText('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') await expect(app.paste('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') await expect(app.pressKey('Return')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') - await expect(app.pressKey('unknown+a')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') + await expect(app.pressKey('', ['command'])).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') expect(f.dispatch).not.toHaveBeenCalled() await expect(window.typeText('window text')).resolves.toEqual(delivered) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'window text' } }) @@ -684,11 +684,7 @@ describe('input actions', () => { ['oversized scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })], ['fractional scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })], ['negative drag point', ({ window }) => window.drag([0, 0], [-1, 3])], - ['empty chord', ({ app }) => app.pressKey('')], - ['missing key', ({ app }) => app.pressKey('super+')], - ['empty plus modifier', ({ app }) => app.pressKey('++')], - ['extra plus delimiter', ({ app }) => app.pressKey('cmd+++')], - ['unknown modifier', ({ app }) => app.pressKey('unknown+a')], + ['empty key', ({ app }) => app.pressKey('')], ] it.each<[string, (surfaces: Surfaces) => Promise]>([ ['getState windowId: Too small: expected number to be >0', ({ app }) => app.getWindow(0)], @@ -714,17 +710,41 @@ describe('input actions', () => { expect(f.dispatch).toHaveBeenCalledTimes(calls) }) - it.each<[string, string, string[]]>([ - ['+', '+', []], - ['cmd++', '+', ['command']], - ['ctrl+shift++', '+', ['control', 'shift']], - ['cmd+plus', 'plus', ['command']], - ['plus', 'plus', []], - ])('dispatches the plus-key chord %s', async (chord, key, modifiers) => { + it.each<[string, string[] | undefined, string[]]>([ + ['+', undefined, []], + ['plus', [], []], + ['+', ['command'], ['command']], + ['Tab', ['shift'], ['shift']], + ])('forwards literal key %s with modifiers %j without splitting', async (key, modifiers, expected) => { const f = fixture() const app = await f.api.getApp('Editor') - await app.pressKey(chord) - expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { appHandle: 'app-17', key, modifiers } }) + if (modifiers === undefined) await app.pressKey(key) + else await app.pressKey(key, modifiers) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { appHandle: 'app-17', key, modifiers: expected } }) + }) + + it('forwards the literal key unsplit in one RPC without mutating modifiers', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + const modifiers: string[] = ['control', 'shift'] + f.dispatch.mockClear() + await app.pressKey('+', modifiers) + expect(f.dispatch).toHaveBeenCalledTimes(1) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { + appHandle: 'app-17', key: '+', modifiers: ['control', 'shift'], + } }) + expect(modifiers).toEqual(['control', 'shift']) + }) + + it('forwards arbitrary provider-defined key and modifier names in one RPC', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockClear() + await app.pressKey('PlatformKey', ['platform-modifier']) + expect(f.dispatch).toHaveBeenCalledTimes(1) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { + appHandle: 'app-17', key: 'PlatformKey', modifiers: ['platform-modifier'], + } }) }) it('passes literal text and keys to the selected app identity', async () => { @@ -733,7 +753,7 @@ describe('input actions', () => { await app.typeText('中文\nline two') await app.paste('粘贴\nplain text') await app.paste('bold', { format: 'html' }) - await app.pressKey('Control_L+shift+a') + await app.pressKey('a', ['control', 'shift']) await app.pressKey('Return') expect(f.dispatch.mock.calls.slice(-5).map(([operation]) => operation)).toEqual([ { method: 'typeText', params: { appHandle: 'app-17', text: '中文\nline two' } }, @@ -753,13 +773,61 @@ describe('input actions', () => { () => window.typeText('hello'), () => window.paste('hello'), () => window.pressKey('Return'), - () => window.pressKey('unknown+a'), + () => window.pressKey('', ['command']), ]) { await expect(run()).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.') } expect(f.dispatch).not.toHaveBeenCalled() }) + it.each<[string, 'down' | 'up']>([ + ['app', 'down'], + ['app', 'up'], + ['window', 'down'], + ['window', 'up'], + ])('sends one explicit %s-scoped key event (%s) with exactly one RPC', async (scope, direction) => { + const f = fixture() + f.setCapabilities({ ...capabilities, keyboard: ['app', 'window'] }) + const app = await f.api.getApp('Editor') + const surface = scope === 'window' ? await app.getWindow(23) : app + f.dispatch.mockClear() + f.dispatch.mockResolvedValueOnce(delivered) + await expect(surface.sendKeyEvent('Shift', direction)).resolves.toEqual(delivered) + expect(f.dispatch).toHaveBeenCalledTimes(1) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'sendKeyEvent', params: { + appHandle: 'app-17', ...scope === 'window' ? { windowId: 23 } : {}, key: 'Shift', direction, + } }) + }) + + it('returns an un-dispatched key-event result from one RPC without retrying', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockClear() + f.dispatch.mockResolvedValueOnce({ dispatched: false }) + await expect(app.sendKeyEvent('Command', 'down')).resolves.toEqual({ dispatched: false }) + expect(f.dispatch).toHaveBeenCalledTimes(1) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'sendKeyEvent', params: { + appHandle: 'app-17', key: 'Command', direction: 'down', + } }) + }) + + it('rejects key events without keyboard support before validating them', async () => { + const appFixture = fixture() + appFixture.setCapabilities({ ...capabilities, keyboard: ['window'] }) + const app = await appFixture.api.getApp('Editor') + appFixture.dispatch.mockClear() + await expect(app.sendKeyEvent('', 'down')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.') + expect(appFixture.dispatch).not.toHaveBeenCalled() + + const windowFixture = fixture() + windowFixture.setCapabilities({ ...capabilities, keyboard: ['app'] }) + const windowApp = await windowFixture.api.getApp('Editor') + const window = await windowApp.getWindow(23) + windowFixture.dispatch.mockClear() + await expect(window.sendKeyEvent('', 'up')).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.') + expect(windowFixture.dispatch).not.toHaveBeenCalled() + }) + it('retains the snapshot identity for element right and double clicks', async () => { const f = fixture() const app = await f.api.getApp('Editor') diff --git a/packages/experimental/cua/tests/native-schema.spec.ts b/packages/experimental/cua/tests/native-schema.spec.ts index 358b87021f..214eaa889c 100644 --- a/packages/experimental/cua/tests/native-schema.spec.ts +++ b/packages/experimental/cua/tests/native-schema.spec.ts @@ -36,6 +36,20 @@ it('admits only locator fields for default invocation', () => { expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId', 'windowId'])) }) +it('admits arbitrary pressKey modifier names without a generated string enum', () => { + const native = nativeProtocol() + expect(native.fields.get('pressKey')).toContain('modifiers') + expect(native.enums.get('pressKey')?.has('modifiers') ?? false).toBe(false) +}) + +it('admits key events with only an explicit direction enum', () => { + const native = nativeProtocol() + expect(native.fields.get('sendKeyEvent')).toEqual(new Set(['appHandle', 'windowId', 'key', 'direction'])) + expect(native.enums.get('sendKeyEvent')?.get('direction')).toEqual(new Set(['down', 'up'])) + expect(native.enums.get('sendKeyEvent')?.has('modifiers') ?? false).toBe(false) + expect(native.enums.get('sendKeyEvent')?.has('key') ?? false).toBe(false) +}) + it('accepts a parameter-free object schema without a properties dictionary', () => { replacements.values.set(0, { properties: { method: { const: 'getCapabilities' }, params: { type: 'object' } } }) expect(nativeProtocol().fields.get('getCapabilities')).toEqual(new Set()) diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index b54654e85f..50fb329c07 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -12,13 +12,27 @@ const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [ { reason: 'numeric element id', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: 0 } } }, { reason: 'coordinate click without a window', request: { method: 'click', params: { appHandle: 'app', x: 1, y: 2 } } }, { reason: 'mixed element and coordinate click', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', windowId: 23, x: 1, y: 2 } } }, - { reason: 'unsupported modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['hyper'] } } }, + { reason: 'non-string pressKey modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['command', 7] } } }, + { reason: 'sendKeyEvent modifiers', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a', direction: 'down', modifiers: [] } } }, + { reason: 'sendKeyEvent without direction', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a' } } }, + { reason: 'sendKeyEvent invalid direction', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a', direction: 'press' } } }, + { reason: 'sendKeyEvent empty key', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: '', direction: 'down' } } }, ] it.each(rejectedRequests)('rejects $reason at operation admission', ({ request }) => { expect(Request.safeParse(request).success).toBe(false) }) +it('admits arbitrary provider-defined pressKey modifier names', () => { + const request = { method: 'pressKey', params: { appHandle: 'app', key: 'PlatformKey', modifiers: ['platform-modifier', 'shift'] } } + expect(Request.parse(request)).toEqual(request) +}) + +it('admits arbitrary key names with an explicit key-event direction', () => { + const request = { method: 'sendKeyEvent', params: { appHandle: 'app', windowId: 23, key: 'Command', direction: 'down' } } + expect(Request.parse(request)).toEqual(request) +}) + it.each([ { appHandle: 'app', snapshotId: 'snapshot', elementId: '0' }, { appHandle: 'app', windowId: 23, x: 1, y: 2 },