feat(cua): separate key modifiers and add directional key events
This commit is contained in:
1 parent
7fc481337d
commit
2dfe4912dd
9 files changed
+192
-60
No files matched your search
@@ -189,6 +189,9 @@ actor AXWorker {
|
||||
trace: ScreenshotTrace? = nil) async throws -> DriverOutcome {
|
||||
try cancellation.check()
|
||||
guard !closed else { throw NativeFailure(code: "NATIVE_CLOSED", message: "Helper is closing") }
|
||||
if method == "sendKeyEvent" {
|
||||
throw NativeFailure(code: "NOT_IMPLEMENTED", message: "sendKeyEvent is not implemented on current platform.")
|
||||
}
|
||||
let locked = observeSession()
|
||||
if method == "getCapabilities" {
|
||||
return .result(.object(["permissions": .object([
|
||||
|
||||
+2
-1
@@ -24,6 +24,7 @@ enum NativeProtocol {
|
||||
"screenshot": ["appHandle", "windowId"],
|
||||
"scroll": ["appHandle", "deltaLinesX", "deltaLinesY", "snapshotId", "windowId", "x", "y"],
|
||||
"selectText": ["appHandle", "elementId", "prefix", "selectionType", "snapshotId", "suffix", "text", "windowId"],
|
||||
"sendKeyEvent": ["appHandle", "direction", "key", "windowId"],
|
||||
"setValue": ["appHandle", "elementId", "snapshotId", "value", "windowId"],
|
||||
"show": ["appHandle", "windowId"],
|
||||
"typeText": ["appHandle", "text", "windowId"],
|
||||
@@ -33,8 +34,8 @@ enum NativeProtocol {
|
||||
"drag": ["modifiers": ["command", "control", "fn", "option", "shift"], "mouseButton": ["left", "middle", "right"]],
|
||||
"openPermissionSettings": ["permission": ["accessibility", "screenRecording"]],
|
||||
"paste": ["format": ["html", "text"]],
|
||||
"pressKey": ["modifiers": ["command", "control", "fn", "option", "shift"]],
|
||||
"requestPermission": ["permission": ["accessibility", "screenRecording"]],
|
||||
"selectText": ["selectionType": ["cursor_after", "cursor_before", "text"]],
|
||||
"sendKeyEvent": ["direction": ["down", "up"]],
|
||||
]
|
||||
}
|
||||
@@ -991,14 +991,7 @@
|
||||
"modifiers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"command",
|
||||
"control",
|
||||
"option",
|
||||
"shift",
|
||||
"fn"
|
||||
]
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1016,6 +1009,55 @@
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"method": {
|
||||
"type": "string",
|
||||
"const": "sendKeyEvent"
|
||||
},
|
||||
"params": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"key": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"direction": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"down",
|
||||
"up"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"key",
|
||||
"direction"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"method",
|
||||
"params"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
@@ -23,31 +23,6 @@ function requestIssueMessage(issues: readonly z.core.$ZodIssue[], parent: readon
|
||||
}))].join('; ')
|
||||
}
|
||||
|
||||
/** Parse a portable key chord.
|
||||
* @param chord Key or chord.
|
||||
* @returns Native key and modifier names.
|
||||
*/
|
||||
export function keyChord(chord: string): { key: string; modifiers: string[] } {
|
||||
const parts = chord.split('+')
|
||||
let key = parts.pop()
|
||||
if (key === '' && parts.at(-1) === '') {
|
||||
key = '+'
|
||||
parts.pop()
|
||||
}
|
||||
if (!key) throw new Error('A key chord must end with a key name')
|
||||
const names: Record<string, string> = {
|
||||
cmd: 'command', command: 'command', super: 'command', Super_L: 'command', Super_R: 'command',
|
||||
ctrl: 'control', control: 'control', Control_L: 'control', Control_R: 'control',
|
||||
alt: 'option', option: 'option', Alt_L: 'option', Alt_R: 'option',
|
||||
shift: 'shift', Shift_L: 'shift', Shift_R: 'shift', fn: 'fn',
|
||||
}
|
||||
return { key, modifiers: parts.map((part) => {
|
||||
const modifier = names[part]
|
||||
if (modifier === undefined) throw new Error(`Unsupported key modifier: ${part}`)
|
||||
return modifier
|
||||
}) }
|
||||
}
|
||||
|
||||
function validatedCall(dispatch: CuaDispatch) {
|
||||
return (method: CuaOperation['method'], params: Record<string, JsonValue>) => {
|
||||
const request = Request.safeParse({ method, params })
|
||||
@@ -188,9 +163,13 @@ function createCuaObjects(call: ReturnType<typeof validatedCall>, sink: CuaObser
|
||||
...options?.format === undefined ? {} : { format: options.format },
|
||||
})
|
||||
},
|
||||
async pressKey(key) {
|
||||
async pressKey(key, modifiers = []) {
|
||||
requireKeyboard()
|
||||
return action('pressKey', keyChord(key))
|
||||
return action('pressKey', { key, modifiers: [...modifiers] })
|
||||
},
|
||||
async sendKeyEvent(key, direction) {
|
||||
requireKeyboard()
|
||||
return action('sendKeyEvent', { key, direction })
|
||||
},
|
||||
}
|
||||
return { surface, call: scopedCall, action, elementParams }
|
||||
|
||||
@@ -61,7 +61,8 @@ export const Request = z.discriminatedUnion('method', [
|
||||
selectionType: z.enum(['text', 'cursor_before', 'cursor_after']).optional() }),
|
||||
operation('typeText', { ...app, windowId: id.optional(), text: z.string() }),
|
||||
operation('paste', { ...app, windowId: id.optional(), text: z.string(), format: z.enum(['text', 'html']).optional() }),
|
||||
operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() }),
|
||||
operation('pressKey', { ...app, windowId: id.optional(), key: z.string().min(1), modifiers: z.array(z.string()).optional() }),
|
||||
operation('sendKeyEvent', { ...app, windowId: id.optional(), key: z.string().min(1), direction: z.enum(['down', 'up']) }),
|
||||
z.object({ method: z.literal('scroll'), params: z.union([
|
||||
z.object({ ...absoluteCoordinate,
|
||||
deltaLinesX: z.number().int().min(-10).max(10), deltaLinesY: z.number().int().min(-10).max(10) }).strict(),
|
||||
|
||||
@@ -160,12 +160,22 @@ export interface CuaSurface {
|
||||
*/
|
||||
paste(text: string, options?: CuaPasteOptions): Promise<CuaActionResult>
|
||||
/**
|
||||
* Send a key or chord at the current input focus.
|
||||
* Press and release one key at the current input focus.
|
||||
* Rejects when the provider does not support keyboard input at this scope.
|
||||
* @param key - xdotool-style chord such as super+c, shift+Tab, + or cmd++; plus is also a key name.
|
||||
* @param key - Provider-defined key name to press and release; not a chord.
|
||||
* @param modifiers - Platform- or provider-defined modifier names held for this keystroke; omitted means none.
|
||||
* @returns Whether the provider successfully executed the requested operation.
|
||||
*/
|
||||
pressKey(key: string): Promise<CuaActionResult>
|
||||
pressKey(key: string, modifiers?: string[]): Promise<CuaActionResult>
|
||||
/**
|
||||
* Send one key-down or key-up event at the current input focus.
|
||||
* Does not send the opposite event.
|
||||
* Rejects when the provider does not support keyboard input at this scope.
|
||||
* @param key - Platform-defined key name; modifier keys may themselves be targets.
|
||||
* @param direction - Explicit event direction; the opposite event is never sent implicitly.
|
||||
* @returns Whether the provider successfully executed the requested operation.
|
||||
*/
|
||||
sendKeyEvent(key: string, direction: 'down' | 'up'): Promise<CuaActionResult>
|
||||
}
|
||||
/**
|
||||
* An explicit window with screenshot and pointer input operations.
|
||||
|
||||
@@ -514,10 +514,10 @@ describe('application and window observations', () => {
|
||||
f.dispatch.mockClear()
|
||||
await window.typeText('hello')
|
||||
await window.paste('<b>bold</b>', { format: 'html' })
|
||||
await window.pressKey('cmd+a')
|
||||
await window.pressKey('a', ['command'])
|
||||
await app.typeText('app text')
|
||||
await app.paste('app paste')
|
||||
await app.pressKey('shift+Tab')
|
||||
await app.pressKey('Tab', ['shift'])
|
||||
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
|
||||
{ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'hello' } },
|
||||
{ method: 'paste', params: { appHandle: 'app-17', windowId: 23, text: '<b>bold</b>', format: 'html' } },
|
||||
@@ -537,7 +537,7 @@ describe('application and window observations', () => {
|
||||
await expect(app.typeText('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
await expect(app.paste('app text')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
await expect(app.pressKey('Return')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
await expect(app.pressKey('unknown+a')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
await expect(app.pressKey('', ['command'])).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
expect(f.dispatch).not.toHaveBeenCalled()
|
||||
await expect(window.typeText('window text')).resolves.toEqual(delivered)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'typeText', params: { appHandle: 'app-17', windowId: 23, text: 'window text' } })
|
||||
@@ -684,11 +684,7 @@ describe('input actions', () => {
|
||||
['oversized scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 11, deltaLinesY: 0 })],
|
||||
['fractional scroll', ({ window }) => window.scroll({ x: 0, y: 0, deltaLinesX: 0.5, deltaLinesY: 0 })],
|
||||
['negative drag point', ({ window }) => window.drag([0, 0], [-1, 3])],
|
||||
['empty chord', ({ app }) => app.pressKey('')],
|
||||
['missing key', ({ app }) => app.pressKey('super+')],
|
||||
['empty plus modifier', ({ app }) => app.pressKey('++')],
|
||||
['extra plus delimiter', ({ app }) => app.pressKey('cmd+++')],
|
||||
['unknown modifier', ({ app }) => app.pressKey('unknown+a')],
|
||||
['empty key', ({ app }) => app.pressKey('')],
|
||||
]
|
||||
it.each<[string, (surfaces: Surfaces) => Promise<unknown>]>([
|
||||
['getState windowId: Too small: expected number to be >0', ({ app }) => app.getWindow(0)],
|
||||
@@ -714,17 +710,41 @@ describe('input actions', () => {
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(calls)
|
||||
})
|
||||
|
||||
it.each<[string, string, string[]]>([
|
||||
['+', '+', []],
|
||||
['cmd++', '+', ['command']],
|
||||
['ctrl+shift++', '+', ['control', 'shift']],
|
||||
['cmd+plus', 'plus', ['command']],
|
||||
['plus', 'plus', []],
|
||||
])('dispatches the plus-key chord %s', async (chord, key, modifiers) => {
|
||||
it.each<[string, string[] | undefined, string[]]>([
|
||||
['+', undefined, []],
|
||||
['plus', [], []],
|
||||
['+', ['command'], ['command']],
|
||||
['Tab', ['shift'], ['shift']],
|
||||
])('forwards literal key %s with modifiers %j without splitting', async (key, modifiers, expected) => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
await app.pressKey(chord)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { appHandle: 'app-17', key, modifiers } })
|
||||
if (modifiers === undefined) await app.pressKey(key)
|
||||
else await app.pressKey(key, modifiers)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { appHandle: 'app-17', key, modifiers: expected } })
|
||||
})
|
||||
|
||||
it('forwards the literal key unsplit in one RPC without mutating modifiers', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
const modifiers: string[] = ['control', 'shift']
|
||||
f.dispatch.mockClear()
|
||||
await app.pressKey('+', modifiers)
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(1)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: {
|
||||
appHandle: 'app-17', key: '+', modifiers: ['control', 'shift'],
|
||||
} })
|
||||
expect(modifiers).toEqual(['control', 'shift'])
|
||||
})
|
||||
|
||||
it('forwards arbitrary provider-defined key and modifier names in one RPC', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
f.dispatch.mockClear()
|
||||
await app.pressKey('PlatformKey', ['platform-modifier'])
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(1)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: {
|
||||
appHandle: 'app-17', key: 'PlatformKey', modifiers: ['platform-modifier'],
|
||||
} })
|
||||
})
|
||||
|
||||
it('passes literal text and keys to the selected app identity', async () => {
|
||||
@@ -733,7 +753,7 @@ describe('input actions', () => {
|
||||
await app.typeText('中文\nline two')
|
||||
await app.paste('粘贴\nplain text')
|
||||
await app.paste('<b>bold</b>', { format: 'html' })
|
||||
await app.pressKey('Control_L+shift+a')
|
||||
await app.pressKey('a', ['control', 'shift'])
|
||||
await app.pressKey('Return')
|
||||
expect(f.dispatch.mock.calls.slice(-5).map(([operation]) => operation)).toEqual([
|
||||
{ method: 'typeText', params: { appHandle: 'app-17', text: '中文\nline two' } },
|
||||
@@ -753,13 +773,61 @@ describe('input actions', () => {
|
||||
() => window.typeText('hello'),
|
||||
() => window.paste('hello'),
|
||||
() => window.pressKey('Return'),
|
||||
() => window.pressKey('unknown+a'),
|
||||
() => window.pressKey('', ['command']),
|
||||
]) {
|
||||
await expect(run()).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.')
|
||||
}
|
||||
expect(f.dispatch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each<[string, 'down' | 'up']>([
|
||||
['app', 'down'],
|
||||
['app', 'up'],
|
||||
['window', 'down'],
|
||||
['window', 'up'],
|
||||
])('sends one explicit %s-scoped key event (%s) with exactly one RPC', async (scope, direction) => {
|
||||
const f = fixture()
|
||||
f.setCapabilities({ ...capabilities, keyboard: ['app', 'window'] })
|
||||
const app = await f.api.getApp('Editor')
|
||||
const surface = scope === 'window' ? await app.getWindow(23) : app
|
||||
f.dispatch.mockClear()
|
||||
f.dispatch.mockResolvedValueOnce(delivered)
|
||||
await expect(surface.sendKeyEvent('Shift', direction)).resolves.toEqual(delivered)
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(1)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'sendKeyEvent', params: {
|
||||
appHandle: 'app-17', ...scope === 'window' ? { windowId: 23 } : {}, key: 'Shift', direction,
|
||||
} })
|
||||
})
|
||||
|
||||
it('returns an un-dispatched key-event result from one RPC without retrying', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
f.dispatch.mockClear()
|
||||
f.dispatch.mockResolvedValueOnce({ dispatched: false })
|
||||
await expect(app.sendKeyEvent('Command', 'down')).resolves.toEqual({ dispatched: false })
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(1)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'sendKeyEvent', params: {
|
||||
appHandle: 'app-17', key: 'Command', direction: 'down',
|
||||
} })
|
||||
})
|
||||
|
||||
it('rejects key events without keyboard support before validating them', async () => {
|
||||
const appFixture = fixture()
|
||||
appFixture.setCapabilities({ ...capabilities, keyboard: ['window'] })
|
||||
const app = await appFixture.api.getApp('Editor')
|
||||
appFixture.dispatch.mockClear()
|
||||
await expect(app.sendKeyEvent('', 'down')).rejects.toThrow('App-scoped keyboard input is not supported on current platform.')
|
||||
expect(appFixture.dispatch).not.toHaveBeenCalled()
|
||||
|
||||
const windowFixture = fixture()
|
||||
windowFixture.setCapabilities({ ...capabilities, keyboard: ['app'] })
|
||||
const windowApp = await windowFixture.api.getApp('Editor')
|
||||
const window = await windowApp.getWindow(23)
|
||||
windowFixture.dispatch.mockClear()
|
||||
await expect(window.sendKeyEvent('', 'up')).rejects.toThrow('Window-scoped keyboard input is not supported on current platform.')
|
||||
expect(windowFixture.dispatch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('retains the snapshot identity for element right and double clicks', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
|
||||
@@ -36,6 +36,20 @@ it('admits only locator fields for default invocation', () => {
|
||||
expect(nativeProtocol().fields.get('invoke')).toEqual(new Set(['appHandle', 'snapshotId', 'elementId', 'windowId']))
|
||||
})
|
||||
|
||||
it('admits arbitrary pressKey modifier names without a generated string enum', () => {
|
||||
const native = nativeProtocol()
|
||||
expect(native.fields.get('pressKey')).toContain('modifiers')
|
||||
expect(native.enums.get('pressKey')?.has('modifiers') ?? false).toBe(false)
|
||||
})
|
||||
|
||||
it('admits key events with only an explicit direction enum', () => {
|
||||
const native = nativeProtocol()
|
||||
expect(native.fields.get('sendKeyEvent')).toEqual(new Set(['appHandle', 'windowId', 'key', 'direction']))
|
||||
expect(native.enums.get('sendKeyEvent')?.get('direction')).toEqual(new Set(['down', 'up']))
|
||||
expect(native.enums.get('sendKeyEvent')?.has('modifiers') ?? false).toBe(false)
|
||||
expect(native.enums.get('sendKeyEvent')?.has('key') ?? false).toBe(false)
|
||||
})
|
||||
|
||||
it('accepts a parameter-free object schema without a properties dictionary', () => {
|
||||
replacements.values.set(0, { properties: { method: { const: 'getCapabilities' }, params: { type: 'object' } } })
|
||||
expect(nativeProtocol().fields.get('getCapabilities')).toEqual(new Set())
|
||||
|
||||
@@ -12,13 +12,27 @@ const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [
|
||||
{ reason: 'numeric element id', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: 0 } } },
|
||||
{ reason: 'coordinate click without a window', request: { method: 'click', params: { appHandle: 'app', x: 1, y: 2 } } },
|
||||
{ reason: 'mixed element and coordinate click', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', windowId: 23, x: 1, y: 2 } } },
|
||||
{ reason: 'unsupported modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['hyper'] } } },
|
||||
{ reason: 'non-string pressKey modifier', request: { method: 'pressKey', params: { appHandle: 'app', key: 'a', modifiers: ['command', 7] } } },
|
||||
{ reason: 'sendKeyEvent modifiers', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a', direction: 'down', modifiers: [] } } },
|
||||
{ reason: 'sendKeyEvent without direction', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a' } } },
|
||||
{ reason: 'sendKeyEvent invalid direction', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: 'a', direction: 'press' } } },
|
||||
{ reason: 'sendKeyEvent empty key', request: { method: 'sendKeyEvent', params: { appHandle: 'app', key: '', direction: 'down' } } },
|
||||
]
|
||||
|
||||
it.each(rejectedRequests)('rejects $reason at operation admission', ({ request }) => {
|
||||
expect(Request.safeParse(request).success).toBe(false)
|
||||
})
|
||||
|
||||
it('admits arbitrary provider-defined pressKey modifier names', () => {
|
||||
const request = { method: 'pressKey', params: { appHandle: 'app', key: 'PlatformKey', modifiers: ['platform-modifier', 'shift'] } }
|
||||
expect(Request.parse(request)).toEqual(request)
|
||||
})
|
||||
|
||||
it('admits arbitrary key names with an explicit key-event direction', () => {
|
||||
const request = { method: 'sendKeyEvent', params: { appHandle: 'app', windowId: 23, key: 'Command', direction: 'down' } }
|
||||
expect(Request.parse(request)).toEqual(request)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0' },
|
||||
{ appHandle: 'app', windowId: 23, x: 1, y: 2 },
|
||||
|
||||
Reference in new issue
Block a user