From bc05190e4a7cbfbedf4a9ee6b120ddab6eb95d94 Mon Sep 17 00:00:00 2001 From: Huo Yaoyuan Date: Mon, 2 Nov 2026 13:43:26 +0800 Subject: [PATCH] feat(cua): perform element actions with optional arguments --- .../native/Sources/CuaNative/AXAccess.swift | 143 ++++++++++++++++++ .../native/Sources/CuaNative/AXWorker.swift | 2 + .../CuaNative/NativeProtocol.generated.swift | 3 +- .../cua-macos/native/protocol.schema.json | 83 ++++++++++ .../tests/swift/PerformActionTests.swift | 93 ++++++++++++ packages/experimental/cua/src/api.ts | 5 + packages/experimental/cua/src/protocol.ts | 1 + packages/experimental/cua/src/types.ts | 8 + .../experimental/cua/src/window-protocol.ts | 1 + packages/experimental/cua/tests/api.spec.ts | 27 ++++ .../experimental/cua/tests/protocol.spec.ts | 19 +++ .../experimental/cua/tests/window-api.spec.ts | 10 +- .../cua-native/tool-schemas.expected.json | 79 ++++++++++ 13 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 packages/experimental/cua-macos/native/tests/swift/PerformActionTests.swift diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift index 1e8cb906b0..b8787f4169 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift @@ -18,6 +18,19 @@ struct AXNodeObservation { let attribute: (String) throws -> CFTypeRef? } +/// Attribute-setting action names are exactly `Role.Attribute.Set`; every other name is a native action. +struct AXAttributeSetTarget: Sendable { + let role: String + let attribute: String + + init?(_ action: String) { + let parts = action.split(separator: ".", omittingEmptySubsequences: false) + guard parts.count == 3, parts[2] == "Set", !parts[0].isEmpty, !parts[1].isEmpty else { return nil } + role = String(parts[0]) + attribute = String(parts[1]) + } +} + final class AXAccess { let timeout: Float let cancellation: CancellationLatch @@ -207,6 +220,136 @@ final class AXAccess { let error = AXUIElementSetAttributeValue(element, "AXValue" as CFString, value as CFString) guard error == .success else { throw failure(error, "set AXValue") } } + /// Perform an action, including attribute-setting operations. + func performAction(_ element: AXUIElement, action: String, argument: JSONValue?) throws { + guard let target = AXAttributeSetTarget(action) else { + guard argument == nil || argument == .null else { + throw RequestParameters.invalid("\(action) is not an attribute-setting action and accepts no argument") + } + try perform(element, action) + return + } + guard let argument, argument != .null else { + throw RequestParameters.invalid("\(target.role).\(target.attribute).Set requires a value") + } + guard let role = try string(element, "AXRole") else { + throw RequestParameters.invalid("Cannot verify an attribute-setting action without an element role") + } + guard role == target.role else { + throw RequestParameters.invalid("Action role \(target.role) does not match element role \(role)") + } + guard try writable(element, target.attribute) == true else { + throw RequestParameters.invalid("\(target.attribute) is not writable on this element") + } + let current = try attribute(element, target.attribute) + let value = try Self.attributeValue(attribute: target.attribute, argument: argument, current: current) + try requireEnabled(element) + try prepare(element) + let error = AXUIElementSetAttributeValue(element, target.attribute as CFString, value) + guard error == .success else { throw failure(error, "set \(target.attribute)") } + } + private enum AttributeValueKind { + case string, boolean, number, point, size, range + } + /// Convert one direct JSON argument to the Core Foundation value expected by the attribute's live type. + static func attributeValue(attribute: String, argument: JSONValue, current: CFTypeRef?) throws -> CFTypeRef { + let kind = try valueKind(attribute: attribute, argument: argument, current: current) + switch kind { + case .string: + guard case .string(let text) = argument else { throw RequestParameters.invalid("\(attribute) requires a string value") } + return text as CFString + case .boolean: + guard case .bool(let flag) = argument else { throw RequestParameters.invalid("\(attribute) requires a boolean value") } + return flag ? kCFBooleanTrue! : kCFBooleanFalse! + case .number: + guard case .number(let number) = argument, number.isFinite else { + throw RequestParameters.invalid("\(attribute) requires a finite number") + } + return NSNumber(value: number) + case .point: + let values = try fields(argument, attribute: attribute, keys: ["x", "y"]) + var point = CGPoint(x: try finite(values["x"], name: "x", attribute: attribute), + y: try finite(values["y"], name: "y", attribute: attribute)) + guard let value = AXValueCreate(.cgPoint, &point) else { + throw NativeFailure(code: "NATIVE_ERROR", message: "Could not create \(attribute)") + } + return value + case .size: + let values = try fields(argument, attribute: attribute, keys: ["width", "height"]) + var size = CGSize(width: try dimension(values["width"], name: "width", attribute: attribute), + height: try dimension(values["height"], name: "height", attribute: attribute)) + guard let value = AXValueCreate(.cgSize, &size) else { + throw NativeFailure(code: "NATIVE_ERROR", message: "Could not create \(attribute)") + } + return value + case .range: + let values = try fields(argument, attribute: attribute, keys: ["location", "length"]) + let location = try nonnegativeInteger(values["location"], name: "location", attribute: attribute) + let length = try nonnegativeInteger(values["length"], name: "length", attribute: attribute) + guard location <= Int.max - length else { throw RequestParameters.invalid("\(attribute) range is too large") } + var range = CFRange(location: location, length: length) + guard let value = AXValueCreate(.cfRange, &range) else { + throw NativeFailure(code: "NATIVE_ERROR", message: "Could not create \(attribute)") + } + return value + } + } + private static func valueKind(attribute: String, argument: JSONValue, current: CFTypeRef?) throws -> AttributeValueKind { + switch attribute { + case "AXPosition": return .point + case "AXSize": return .size + case "AXSelectedTextRange": return .range + default: break + } + if let current { return try inferredKind(current, attribute: attribute) } + switch argument { + case .string: return .string + case .bool: return .boolean + case .number: return .number + default: + throw NativeFailure(code: "UNSUPPORTED", message: "Accessibility \(attribute) has no known value type for this JSON argument") + } + } + private static func inferredKind(_ value: CFTypeRef, attribute: String) throws -> AttributeValueKind { + let type = CFGetTypeID(value) + if type == CFStringGetTypeID() { return .string } + if type == CFBooleanGetTypeID() { return .boolean } + if type == CFNumberGetTypeID() { return .number } + if type == AXValueGetTypeID() { + switch AXValueGetType(unsafeDowncast(value, to: AXValue.self)) { + case .cgPoint: return .point + case .cgSize: return .size + case .cfRange: return .range + default: break + } + } + throw NativeFailure(code: "UNSUPPORTED", message: "Accessibility \(attribute) value type is not supported for attribute-setting arguments") + } + private static func fields(_ argument: JSONValue, attribute: String, keys: Set) throws -> [String: JSONValue] { + guard case .object(let values) = argument, Set(values.keys) == keys else { + throw RequestParameters.invalid("\(attribute) requires exactly \(keys.sorted().joined(separator: ", "))") + } + return values + } + private static func finite(_ value: JSONValue?, name: String, attribute: String) throws -> Double { + guard let value, case .number(let number) = value, number.isFinite else { + throw RequestParameters.invalid("\(attribute) \(name) must be a finite number") + } + return number + } + private static func dimension(_ value: JSONValue?, name: String, attribute: String) throws -> Double { + let number = try finite(value, name: name, attribute: attribute) + guard number >= 0 else { throw RequestParameters.invalid("\(attribute) \(name) must be nonnegative") } + return number + } + private static func nonnegativeInteger(_ value: JSONValue?, name: String, attribute: String) throws -> Int { + guard let value, case .number(let number) = value, number.isFinite, + number.rounded(.towardZero) == number, number >= 0, number <= Double(Int.max), + let integer = Int(exactly: number) else { + throw RequestParameters.invalid("\(attribute) \(name) must be a nonnegative integer") + } + return integer + } func pasteCommand(_ pid: Int32, maxElements: Int, maxDepth: Int) throws -> AXUIElement { guard let menu = try element(AXUIElementCreateApplication(pid), "AXMenuBar") else { throw NativeFailure(code: "UNSUPPORTED", message: "The application does not expose a Paste menu command") diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index fc0779a392..842c5b8bc8 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -313,6 +313,8 @@ actor AXWorker { if method == "typeText" { try inputs().0.typeText(params.string("text"), to: recipient.pid) } else { try inputs().0.pressKey(params.string("key"), modifiers: params.modifiers(), to: recipient.pid) } case "setValue": try ax().setValue(element(handle, params), params.string("value")) + case "performAction": + try ax().performAction(element(handle, params), action: params.string("action"), argument: params.values["argument"]) case "selectText": try ax().selectText(element(handle, params), text: params.string("text"), prefix: params.optionalString("prefix"), suffix: params.optionalString("suffix"), mode: params.optionalString("selectionType") ?? "text") diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift index a170b4cf14..485dc105dc 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift @@ -1,6 +1,6 @@ // Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit. enum NativeProtocol { - static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listActions", "listApps", "listWindows", "paste", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"] + static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listActions", "listApps", "listWindows", "paste", "performAction", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"] static let fields: [String: Set] = [ "activate": ["appHandle"], "applicationIcons": ["paths"], @@ -17,6 +17,7 @@ enum NativeProtocol { "listWindows": ["appHandle"], "openPermissionSettings": ["permission"], "paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text"], + "performAction": ["action", "appHandle", "argument", "elementId", "snapshotId"], "permissionStatus": [], "pressKey": ["appHandle", "key", "modifiers"], "requestPermission": ["permission"], diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index 578b140385..b015d70399 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -315,6 +315,89 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "method": { + "type": "string", + "const": "performAction" + }, + "params": { + "type": "object", + "properties": { + "appHandle": { + "type": "string", + "minLength": 1 + }, + "snapshotId": { + "type": "string", + "minLength": 1 + }, + "elementId": { + "type": "string", + "minLength": 1, + "description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot." + }, + "action": { + "type": "string", + "minLength": 1 + }, + "argument": { + "$ref": "#/$defs/__schema0" + } + }, + "required": [ + "appHandle", + "snapshotId", + "elementId", + "action" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "method", + "params" + ], + "additionalProperties": false, + "$defs": { + "__schema0": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/__schema0" + } + }, + { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "$ref": "#/$defs/__schema0" + } + } + ] + } + } + }, { "type": "object", "properties": { diff --git a/packages/experimental/cua-macos/native/tests/swift/PerformActionTests.swift b/packages/experimental/cua-macos/native/tests/swift/PerformActionTests.swift new file mode 100644 index 0000000000..263601029f --- /dev/null +++ b/packages/experimental/cua-macos/native/tests/swift/PerformActionTests.swift @@ -0,0 +1,93 @@ +// Attribute-setting dispatch accepts only complete Role.Attribute.Set names and direct JSON values. +import ApplicationServices +import CoreGraphics +import Foundation +import Testing +@testable import CuaNative + +@Test func parsesOnlyCompleteAttributeSetActions() { + let target = AXAttributeSetTarget("AXTextArea.AXValue.Set") + #expect(target?.role == "AXTextArea") + #expect(target?.attribute == "AXValue") + for action in [ + "AXPress", + "AXTextArea.AXValue.set", + "AXTextArea.AXValue.SET", + "AXTextArea.AXValue.Set.Extra", + "AXTextArea..Set", + ".AXValue.Set", + "AXTextArea.AXValue.", + ] { + #expect(AXAttributeSetTarget(action) == nil) + } +} + +@Test func mapsScalarsToTheirDirectCoreFoundationTypes() throws { + #expect(CFGetTypeID(try AXAccess.attributeValue(attribute: "AXValue", argument: .string("hello"), current: nil)) == CFStringGetTypeID()) + #expect(CFGetTypeID(try AXAccess.attributeValue(attribute: "AXValue", argument: .bool(true), current: nil)) == CFBooleanGetTypeID()) + #expect(CFGetTypeID(try AXAccess.attributeValue(attribute: "AXValue", argument: .number(2.5), current: nil)) == CFNumberGetTypeID()) +} + +@Test func mapsPositionSizeAndRangeObjectsToAccessibilityValues() throws { + let position = unsafeDowncast(try AXAccess.attributeValue(attribute: "AXPosition", + argument: .object(["x": .number(10), "y": .number(20)]), current: nil), to: AXValue.self) + var point = CGPoint.zero + #expect(AXValueGetType(position) == .cgPoint) + #expect(AXValueGetValue(position, .cgPoint, &point)) + #expect(point == CGPoint(x: 10, y: 20)) + + let size = unsafeDowncast(try AXAccess.attributeValue(attribute: "AXSize", + argument: .object(["width": .number(640), "height": .number(480)]), current: nil), to: AXValue.self) + var dimensions = CGSize.zero + #expect(AXValueGetType(size) == .cgSize) + #expect(AXValueGetValue(size, .cgSize, &dimensions)) + #expect(dimensions == CGSize(width: 640, height: 480)) + + let range = unsafeDowncast(try AXAccess.attributeValue(attribute: "AXSelectedTextRange", + argument: .object(["location": .number(2), "length": .number(3)]), current: nil), to: AXValue.self) + var value = CFRange() + #expect(AXValueGetType(range) == .cfRange) + #expect(AXValueGetValue(range, .cfRange, &value)) + #expect(value.location == 2) + #expect(value.length == 3) +} + +@Test func rejectsArgumentsForPlainActionsAndMissingAttributeValues() { + let access = AXAccess(timeout: 1, cancellation: CancellationLatch()) + let element = AXUIElementCreateApplication(0) + for argument in [JSONValue.string("value"), .bool(true), .number(1), .array([]), .object([:])] { + do { + try access.performAction(element, action: "AXPress", argument: argument) + Issue.record("AXPress accepted an argument") + } catch let error as NativeFailure { + #expect(error.code == "INVALID_ARGUMENT") + } catch { Issue.record("Unexpected error: \(error)") } + } + let absent: [JSONValue?] = [nil, .null] + for argument in absent { + do { + try access.performAction(element, action: "AXTextArea.AXValue.Set", argument: argument) + Issue.record("Attribute setting accepted no value") + } catch let error as NativeFailure { + #expect(error.code == "INVALID_ARGUMENT") + } catch { Issue.record("Unexpected error: \(error)") } + } +} + +@Test func rejectsUnsupportedOrMalformedAttributeValues() { + let cases: [(String, JSONValue, CFTypeRef?)] = [ + ("AXPosition", .object(["x": .number(1)]), nil), + ("AXSize", .object(["width": .number(-1), "height": .number(1)]), nil), + ("AXSelectedTextRange", .object(["location": .number(0.5), "length": .number(1)]), nil), + ("AXChildren", .object(["value": .string("x")]), nil), + ("AXChildren", .array([]), [String]() as CFArray), + ] + for (attribute, argument, current) in cases { + do { + _ = try AXAccess.attributeValue(attribute: attribute, argument: argument, current: current) + Issue.record("\(attribute) accepted \(argument)") + } catch let error as NativeFailure { + #expect(error.code == "INVALID_ARGUMENT" || error.code == "UNSUPPORTED") + } catch { Issue.record("Unexpected error: \(error)") } + } +} diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index 7e5b9c9889..e112dd1a52 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -192,6 +192,11 @@ export function createCuaSurface(dispatch: CuaDispatch, sink: CuaObservationSink async listActions(elementId) { return ActionNamesResult.parse(await call('listActions', { ...snapshotParams(), elementId })) }, + async performAction(elementId, name, argument) { + return action('performAction', { ...snapshotParams(), elementId, action: name, + ...argument === undefined ? {} : { argument }, + }) + }, async hover(target, options) { const relative = options?.relative === undefined ? {} : { relative: options.relative } if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...snapshotParams(), elementId: target, ...relative })) diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index a304ae9de3..92928937bd 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -45,6 +45,7 @@ export const Request = z.discriminatedUnion('method', [ details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'), }), operation('listActions', snapshot), + operation('performAction', { ...snapshot, action: z.string().min(1), argument: z.json().optional() }), z.object({ method: z.literal('hover'), params: z.union([ z.object(snapshot).strict(), z.object(absoluteCoordinate).strict(), z.object(relativeCoordinate).strict(), ]) }).strict(), diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 5045a64478..1c3fc3e496 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -123,6 +123,14 @@ export interface CuaSurface { * @returns Provider-defined action names; empty when the element advertises none. */ listActions(elementId: CuaElementId): Promise + /** + * Perform an element action, including attribute-setting operations. + * @param elementId - Element id from this surface's latest observation. + * @param action - Exact action name from listActions. + * @param argument - Direct action argument; omit or pass null for a no-argument action, or pass the value itself for attribute setting. + * @returns Dispatch evidence; the application state is not verified. + */ + performAction(elementId: CuaElementId, action: string, argument?: JsonValue): Promise /** * Reveal advertised AX alternate UI, otherwise move the system cursor without pressing a button. * The system-pointer route requires the app to be foreground and the point unobscured; never activates or raises it. diff --git a/packages/experimental/cua/src/window-protocol.ts b/packages/experimental/cua/src/window-protocol.ts index fd54561ee1..0ae98f1230 100644 --- a/packages/experimental/cua/src/window-protocol.ts +++ b/packages/experimental/cua/src/window-protocol.ts @@ -26,6 +26,7 @@ export const WindowRequest = z.discriminatedUnion('method', [ operation('launch', { target: z.string().min(1) }), operation('getState', { ...handle, includeMenus: z.boolean().optional(), details: z.boolean().optional() }), operation('listActions', snapshot), + operation('performAction', { ...snapshot, action: z.string().min(1), argument: z.json().optional() }), operation('activate', handle), operation('show', handle), operation('screenshot', handle), operation('typeText', { ...handle, text: z.string() }), operation('paste', { ...handle, text: z.string(), format: z.enum(['text', 'html']).optional() }), diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index cbf4035151..f126cf01e0 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -248,6 +248,33 @@ describe('element action queries', () => { }) }) +describe('element actions', () => { + it('dispatches advertised actions with omitted, null or direct values without wrapping the value', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockClear() + expect(await app.performAction(element('42.12345'), 'AXPress')).toEqual(delivered) + expect(await app.performAction(element('42.12345'), 'AXPress', null)).toEqual(delivered) + expect(await app.performAction(element('42.12345'), 'AXTextArea.AXValue.Set', 'hello')).toEqual(delivered) + expect(await app.performAction(element('42.12345'), 'AXWindow.AXPosition.Set', { x: 10, y: 20 })).toEqual(delivered) + expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([ + { method: 'performAction', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345', action: 'AXPress' } }, + { method: 'performAction', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345', action: 'AXPress', argument: null } }, + { method: 'performAction', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345', action: 'AXTextArea.AXValue.Set', argument: 'hello' } }, + { method: 'performAction', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345', action: 'AXWindow.AXPosition.Set', argument: { x: 10, y: 20 } } }, + ]) + expect(f.sink.text).not.toHaveBeenCalled() + }) + + it('rejects an empty action before dispatch', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockClear() + await expect(app.performAction(element('0'), '')).rejects.toThrow('performAction action') + expect(f.dispatch).not.toHaveBeenCalled() + }) +}) + describe('application and window observations', () => { it('keeps the latest element id scope separate for an app and each selected window', async () => { const f = fixture() diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index 8f05dd6721..812e16dd41 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -94,6 +94,25 @@ it('admits an element action query only with a retained snapshot identity', () = ]) expect(Request.safeParse({ method: 'listActions', params: invalid }).success).toBe(false) }) +it.each([ + { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXPress' }, + { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXPress', argument: null }, + { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXTextArea.AXValue.Set', argument: 'hello' }, + { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXTextArea.AXSelectedTextRange.Set', argument: { location: 0, length: 3 } }, + { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXWindow.AXPosition.Set', argument: { x: 10, y: 20 } }, +])('admits performAction $action with its direct argument', (params) => { + expect(Request.parse({ method: 'performAction', params })).toEqual({ method: 'performAction', params }) +}) + +it('requires a retained snapshot and a nonempty action for performAction', () => { + for (const invalid of [ + { appHandle: 'app', snapshotId: 'snapshot', elementId: '0' }, + { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: '' }, + { appHandle: 'app', elementId: '0', action: 'AXPress' }, + { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXPress', extra: true }, + ]) expect(Request.safeParse({ method: 'performAction', params: invalid }).success).toBe(false) +}) + it.each(['click', 'hover', 'scroll', 'drag'])('validates 0–999 integer coordinates for %s', (method) => { const params = { appHandle: 'app', windowId: 7, relative: true, ...method === 'drag' ? { fromX: 0, fromY: 0, toX: 999, toY: 999 } : { x: 0, y: 999 }, diff --git a/packages/experimental/cua/tests/window-api.spec.ts b/packages/experimental/cua/tests/window-api.spec.ts index fc2fc4e761..be03c8d7eb 100644 --- a/packages/experimental/cua/tests/window-api.spec.ts +++ b/packages/experimental/cua/tests/window-api.spec.ts @@ -52,13 +52,21 @@ it('binds all observation and keyboard operations to a selected window and suppr expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'listActions', params: { windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345', } }) + await window.performAction(element('42.12345'), 'AXTextArea.AXValue.Set', 'hello') + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'performAction', params: { + windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345', + action: 'AXTextArea.AXValue.Set', argument: 'hello', + } }) }) -it('accepts an element action query only for a selected window snapshot', () => { +it('accepts an element action query or execution only for a selected window snapshot', () => { const params = { windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345' } expect(WindowRequest.parse({ method: 'listActions', params })).toEqual({ method: 'listActions', params }) expect(WindowRequest.safeParse({ method: 'listActions', params: { windowHandle: 'window-7', elementId: '42.12345', } }).success).toBe(false) + const action = { ...params, action: 'AXTextArea.AXValue.Set', argument: 'hello' } + expect(WindowRequest.parse({ method: 'performAction', params: action })).toEqual({ method: 'performAction', params: action }) + expect(WindowRequest.safeParse({ method: 'performAction', params: { ...params, action: '' } }).success).toBe(false) }) it('launches a discovered entry without inventing a selected window or waiting for one', async () => { const f = fixture() diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index f74f96338d..5d40f031f3 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -328,6 +328,52 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "performAction" + }, + "params": { + "type": "object", + "properties": { + "appHandle": { + "type": "string", + "minLength": 1 + }, + "snapshotId": { + "type": "string", + "minLength": 1 + }, + "elementId": { + "description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot.", + "type": "string", + "minLength": 1 + }, + "action": { + "type": "string", + "minLength": 1 + }, + "argument": { + "$ref": "#/$defs/__schema0" + } + }, + "required": [ + "appHandle", + "snapshotId", + "elementId", + "action" + ], + "additionalProperties": false + } + }, + "required": [ + "method", + "params" + ], + "additionalProperties": false + }, { "type": "object", "properties": { @@ -1258,6 +1304,39 @@ "additionalProperties": false } ], + "$defs": { + "__schema0": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/__schema0" + } + }, + { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "$ref": "#/$defs/__schema0" + } + } + ] + } + }, "type": "object" } },