feat(cua): perform element actions with optional arguments

This commit is contained in:
Huo Yaoyuan committed 2026-11-02 13:43:26 +08:00
1 parent b0d175ad9d
commit bc05190e4a
13 files changed
+472 -2

No files matched your search

@@ -18,6 +18,19 @@ struct AXNodeObservation {
let attribute: (String) throws -> CFTypeRef?
}
/// Attribute-setting action names are exactly `Role.Attribute.Set`; every other name is a native action.
struct AXAttributeSetTarget: Sendable {
let role: String
let attribute: String
init?(_ action: String) {
let parts = action.split(separator: ".", omittingEmptySubsequences: false)
guard parts.count == 3, parts[2] == "Set", !parts[0].isEmpty, !parts[1].isEmpty else { return nil }
role = String(parts[0])
attribute = String(parts[1])
}
}
final class AXAccess {
let timeout: Float
let cancellation: CancellationLatch
@@ -207,6 +220,136 @@ final class AXAccess {
let error = AXUIElementSetAttributeValue(element, "AXValue" as CFString, value as CFString)
guard error == .success else { throw failure(error, "set AXValue") }
}
/// Perform an action, including attribute-setting operations.
func performAction(_ element: AXUIElement, action: String, argument: JSONValue?) throws {
guard let target = AXAttributeSetTarget(action) else {
guard argument == nil || argument == .null else {
throw RequestParameters.invalid("\(action) is not an attribute-setting action and accepts no argument")
}
try perform(element, action)
return
}
guard let argument, argument != .null else {
throw RequestParameters.invalid("\(target.role).\(target.attribute).Set requires a value")
}
guard let role = try string(element, "AXRole") else {
throw RequestParameters.invalid("Cannot verify an attribute-setting action without an element role")
}
guard role == target.role else {
throw RequestParameters.invalid("Action role \(target.role) does not match element role \(role)")
}
guard try writable(element, target.attribute) == true else {
throw RequestParameters.invalid("\(target.attribute) is not writable on this element")
}
let current = try attribute(element, target.attribute)
let value = try Self.attributeValue(attribute: target.attribute, argument: argument, current: current)
try requireEnabled(element)
try prepare(element)
let error = AXUIElementSetAttributeValue(element, target.attribute as CFString, value)
guard error == .success else { throw failure(error, "set \(target.attribute)") }
}
private enum AttributeValueKind {
case string, boolean, number, point, size, range
}
/// Convert one direct JSON argument to the Core Foundation value expected by the attribute's live type.
static func attributeValue(attribute: String, argument: JSONValue, current: CFTypeRef?) throws -> CFTypeRef {
let kind = try valueKind(attribute: attribute, argument: argument, current: current)
switch kind {
case .string:
guard case .string(let text) = argument else { throw RequestParameters.invalid("\(attribute) requires a string value") }
return text as CFString
case .boolean:
guard case .bool(let flag) = argument else { throw RequestParameters.invalid("\(attribute) requires a boolean value") }
return flag ? kCFBooleanTrue! : kCFBooleanFalse!
case .number:
guard case .number(let number) = argument, number.isFinite else {
throw RequestParameters.invalid("\(attribute) requires a finite number")
}
return NSNumber(value: number)
case .point:
let values = try fields(argument, attribute: attribute, keys: ["x", "y"])
var point = CGPoint(x: try finite(values["x"], name: "x", attribute: attribute),
y: try finite(values["y"], name: "y", attribute: attribute))
guard let value = AXValueCreate(.cgPoint, &point) else {
throw NativeFailure(code: "NATIVE_ERROR", message: "Could not create \(attribute)")
}
return value
case .size:
let values = try fields(argument, attribute: attribute, keys: ["width", "height"])
var size = CGSize(width: try dimension(values["width"], name: "width", attribute: attribute),
height: try dimension(values["height"], name: "height", attribute: attribute))
guard let value = AXValueCreate(.cgSize, &size) else {
throw NativeFailure(code: "NATIVE_ERROR", message: "Could not create \(attribute)")
}
return value
case .range:
let values = try fields(argument, attribute: attribute, keys: ["location", "length"])
let location = try nonnegativeInteger(values["location"], name: "location", attribute: attribute)
let length = try nonnegativeInteger(values["length"], name: "length", attribute: attribute)
guard location <= Int.max - length else { throw RequestParameters.invalid("\(attribute) range is too large") }
var range = CFRange(location: location, length: length)
guard let value = AXValueCreate(.cfRange, &range) else {
throw NativeFailure(code: "NATIVE_ERROR", message: "Could not create \(attribute)")
}
return value
}
}
private static func valueKind(attribute: String, argument: JSONValue, current: CFTypeRef?) throws -> AttributeValueKind {
switch attribute {
case "AXPosition": return .point
case "AXSize": return .size
case "AXSelectedTextRange": return .range
default: break
}
if let current { return try inferredKind(current, attribute: attribute) }
switch argument {
case .string: return .string
case .bool: return .boolean
case .number: return .number
default:
throw NativeFailure(code: "UNSUPPORTED", message: "Accessibility \(attribute) has no known value type for this JSON argument")
}
}
private static func inferredKind(_ value: CFTypeRef, attribute: String) throws -> AttributeValueKind {
let type = CFGetTypeID(value)
if type == CFStringGetTypeID() { return .string }
if type == CFBooleanGetTypeID() { return .boolean }
if type == CFNumberGetTypeID() { return .number }
if type == AXValueGetTypeID() {
switch AXValueGetType(unsafeDowncast(value, to: AXValue.self)) {
case .cgPoint: return .point
case .cgSize: return .size
case .cfRange: return .range
default: break
}
}
throw NativeFailure(code: "UNSUPPORTED", message: "Accessibility \(attribute) value type is not supported for attribute-setting arguments")
}
private static func fields(_ argument: JSONValue, attribute: String, keys: Set<String>) throws -> [String: JSONValue] {
guard case .object(let values) = argument, Set(values.keys) == keys else {
throw RequestParameters.invalid("\(attribute) requires exactly \(keys.sorted().joined(separator: ", "))")
}
return values
}
private static func finite(_ value: JSONValue?, name: String, attribute: String) throws -> Double {
guard let value, case .number(let number) = value, number.isFinite else {
throw RequestParameters.invalid("\(attribute) \(name) must be a finite number")
}
return number
}
private static func dimension(_ value: JSONValue?, name: String, attribute: String) throws -> Double {
let number = try finite(value, name: name, attribute: attribute)
guard number >= 0 else { throw RequestParameters.invalid("\(attribute) \(name) must be nonnegative") }
return number
}
private static func nonnegativeInteger(_ value: JSONValue?, name: String, attribute: String) throws -> Int {
guard let value, case .number(let number) = value, number.isFinite,
number.rounded(.towardZero) == number, number >= 0, number <= Double(Int.max),
let integer = Int(exactly: number) else {
throw RequestParameters.invalid("\(attribute) \(name) must be a nonnegative integer")
}
return integer
}
func pasteCommand(_ pid: Int32, maxElements: Int, maxDepth: Int) throws -> AXUIElement {
guard let menu = try element(AXUIElementCreateApplication(pid), "AXMenuBar") else {
throw NativeFailure(code: "UNSUPPORTED", message: "The application does not expose a Paste menu command")
@@ -313,6 +313,8 @@ actor AXWorker {
if method == "typeText" { try inputs().0.typeText(params.string("text"), to: recipient.pid) }
else { try inputs().0.pressKey(params.string("key"), modifiers: params.modifiers(), to: recipient.pid) }
case "setValue": try ax().setValue(element(handle, params), params.string("value"))
case "performAction":
try ax().performAction(element(handle, params), action: params.string("action"), argument: params.values["argument"])
case "selectText":
try ax().selectText(element(handle, params), text: params.string("text"), prefix: params.optionalString("prefix"),
suffix: params.optionalString("suffix"), mode: params.optionalString("selectionType") ?? "text")
@@ -1,6 +1,6 @@
// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.
enum NativeProtocol {
static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listActions", "listApps", "listWindows", "paste", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"]
static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listActions", "listApps", "listWindows", "paste", "performAction", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"]
static let fields: [String: Set<String>] = [
"activate": ["appHandle"],
"applicationIcons": ["paths"],
@@ -17,6 +17,7 @@ enum NativeProtocol {
"listWindows": ["appHandle"],
"openPermissionSettings": ["permission"],
"paste": ["appHandle", "format", "maxClipboardBytes", "maxDepth", "maxElements", "pasteReadTimeoutMs", "text"],
"performAction": ["action", "appHandle", "argument", "elementId", "snapshotId"],
"permissionStatus": [],
"pressKey": ["appHandle", "key", "modifiers"],
"requestPermission": ["permission"],
@@ -315,6 +315,89 @@
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"id": {
"type": "string"
},
"method": {
"type": "string",
"const": "performAction"
},
"params": {
"type": "object",
"properties": {
"appHandle": {
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
"minLength": 1
},
"elementId": {
"type": "string",
"minLength": 1,
"description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot."
},
"action": {
"type": "string",
"minLength": 1
},
"argument": {
"$ref": "#/$defs/__schema0"
}
},
"required": [
"appHandle",
"snapshotId",
"elementId",
"action"
],
"additionalProperties": false
}
},
"required": [
"id",
"method",
"params"
],
"additionalProperties": false,
"$defs": {
"__schema0": {
"anyOf": [
{
"type": "string"
},
{
"type": "number"
},
{
"type": "boolean"
},
{
"type": "null"
},
{
"type": "array",
"items": {
"$ref": "#/$defs/__schema0"
}
},
{
"type": "object",
"propertyNames": {
"type": "string"
},
"additionalProperties": {
"$ref": "#/$defs/__schema0"
}
}
]
}
}
},
{
"type": "object",
"properties": {
@@ -0,0 +1,93 @@
// Attribute-setting dispatch accepts only complete Role.Attribute.Set names and direct JSON values.
import ApplicationServices
import CoreGraphics
import Foundation
import Testing
@testable import CuaNative
@Test func parsesOnlyCompleteAttributeSetActions() {
let target = AXAttributeSetTarget("AXTextArea.AXValue.Set")
#expect(target?.role == "AXTextArea")
#expect(target?.attribute == "AXValue")
for action in [
"AXPress",
"AXTextArea.AXValue.set",
"AXTextArea.AXValue.SET",
"AXTextArea.AXValue.Set.Extra",
"AXTextArea..Set",
".AXValue.Set",
"AXTextArea.AXValue.",
] {
#expect(AXAttributeSetTarget(action) == nil)
}
}
@Test func mapsScalarsToTheirDirectCoreFoundationTypes() throws {
#expect(CFGetTypeID(try AXAccess.attributeValue(attribute: "AXValue", argument: .string("hello"), current: nil)) == CFStringGetTypeID())
#expect(CFGetTypeID(try AXAccess.attributeValue(attribute: "AXValue", argument: .bool(true), current: nil)) == CFBooleanGetTypeID())
#expect(CFGetTypeID(try AXAccess.attributeValue(attribute: "AXValue", argument: .number(2.5), current: nil)) == CFNumberGetTypeID())
}
@Test func mapsPositionSizeAndRangeObjectsToAccessibilityValues() throws {
let position = unsafeDowncast(try AXAccess.attributeValue(attribute: "AXPosition",
argument: .object(["x": .number(10), "y": .number(20)]), current: nil), to: AXValue.self)
var point = CGPoint.zero
#expect(AXValueGetType(position) == .cgPoint)
#expect(AXValueGetValue(position, .cgPoint, &point))
#expect(point == CGPoint(x: 10, y: 20))
let size = unsafeDowncast(try AXAccess.attributeValue(attribute: "AXSize",
argument: .object(["width": .number(640), "height": .number(480)]), current: nil), to: AXValue.self)
var dimensions = CGSize.zero
#expect(AXValueGetType(size) == .cgSize)
#expect(AXValueGetValue(size, .cgSize, &dimensions))
#expect(dimensions == CGSize(width: 640, height: 480))
let range = unsafeDowncast(try AXAccess.attributeValue(attribute: "AXSelectedTextRange",
argument: .object(["location": .number(2), "length": .number(3)]), current: nil), to: AXValue.self)
var value = CFRange()
#expect(AXValueGetType(range) == .cfRange)
#expect(AXValueGetValue(range, .cfRange, &value))
#expect(value.location == 2)
#expect(value.length == 3)
}
@Test func rejectsArgumentsForPlainActionsAndMissingAttributeValues() {
let access = AXAccess(timeout: 1, cancellation: CancellationLatch())
let element = AXUIElementCreateApplication(0)
for argument in [JSONValue.string("value"), .bool(true), .number(1), .array([]), .object([:])] {
do {
try access.performAction(element, action: "AXPress", argument: argument)
Issue.record("AXPress accepted an argument")
} catch let error as NativeFailure {
#expect(error.code == "INVALID_ARGUMENT")
} catch { Issue.record("Unexpected error: \(error)") }
}
let absent: [JSONValue?] = [nil, .null]
for argument in absent {
do {
try access.performAction(element, action: "AXTextArea.AXValue.Set", argument: argument)
Issue.record("Attribute setting accepted no value")
} catch let error as NativeFailure {
#expect(error.code == "INVALID_ARGUMENT")
} catch { Issue.record("Unexpected error: \(error)") }
}
}
@Test func rejectsUnsupportedOrMalformedAttributeValues() {
let cases: [(String, JSONValue, CFTypeRef?)] = [
("AXPosition", .object(["x": .number(1)]), nil),
("AXSize", .object(["width": .number(-1), "height": .number(1)]), nil),
("AXSelectedTextRange", .object(["location": .number(0.5), "length": .number(1)]), nil),
("AXChildren", .object(["value": .string("x")]), nil),
("AXChildren", .array([]), [String]() as CFArray),
]
for (attribute, argument, current) in cases {
do {
_ = try AXAccess.attributeValue(attribute: attribute, argument: argument, current: current)
Issue.record("\(attribute) accepted \(argument)")
} catch let error as NativeFailure {
#expect(error.code == "INVALID_ARGUMENT" || error.code == "UNSUPPORTED")
} catch { Issue.record("Unexpected error: \(error)") }
}
}
+5
View File
@@ -192,6 +192,11 @@ export function createCuaSurface(dispatch: CuaDispatch, sink: CuaObservationSink
async listActions(elementId) {
return ActionNamesResult.parse(await call('listActions', { ...snapshotParams(), elementId }))
},
async performAction(elementId, name, argument) {
return action('performAction', { ...snapshotParams(), elementId, action: name,
...argument === undefined ? {} : { argument },
})
},
async hover(target, options) {
const relative = options?.relative === undefined ? {} : { relative: options.relative }
if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...snapshotParams(), elementId: target, ...relative }))
@@ -45,6 +45,7 @@ export const Request = z.discriminatedUnion('method', [
details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'),
}),
operation('listActions', snapshot),
operation('performAction', { ...snapshot, action: z.string().min(1), argument: z.json().optional() }),
z.object({ method: z.literal('hover'), params: z.union([
z.object(snapshot).strict(), z.object(absoluteCoordinate).strict(), z.object(relativeCoordinate).strict(),
]) }).strict(),
+8
View File
@@ -123,6 +123,14 @@ export interface CuaSurface {
* @returns Provider-defined action names; empty when the element advertises none.
*/
listActions(elementId: CuaElementId): Promise<string[]>
/**
* Perform an element action, including attribute-setting operations.
* @param elementId - Element id from this surface's latest observation.
* @param action - Exact action name from listActions.
* @param argument - Direct action argument; omit or pass null for a no-argument action, or pass the value itself for attribute setting.
* @returns Dispatch evidence; the application state is not verified.
*/
performAction(elementId: CuaElementId, action: string, argument?: JsonValue): Promise<CuaActionResult>
/**
* Reveal advertised AX alternate UI, otherwise move the system cursor without pressing a button.
* The system-pointer route requires the app to be foreground and the point unobscured; never activates or raises it.
@@ -26,6 +26,7 @@ export const WindowRequest = z.discriminatedUnion('method', [
operation('launch', { target: z.string().min(1) }),
operation('getState', { ...handle, includeMenus: z.boolean().optional(), details: z.boolean().optional() }),
operation('listActions', snapshot),
operation('performAction', { ...snapshot, action: z.string().min(1), argument: z.json().optional() }),
operation('activate', handle), operation('show', handle), operation('screenshot', handle),
operation('typeText', { ...handle, text: z.string() }),
operation('paste', { ...handle, text: z.string(), format: z.enum(['text', 'html']).optional() }),
@@ -248,6 +248,33 @@ describe('element action queries', () => {
})
})
describe('element actions', () => {
it('dispatches advertised actions with omitted, null or direct values without wrapping the value', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockClear()
expect(await app.performAction(element('42.12345'), 'AXPress')).toEqual(delivered)
expect(await app.performAction(element('42.12345'), 'AXPress', null)).toEqual(delivered)
expect(await app.performAction(element('42.12345'), 'AXTextArea.AXValue.Set', 'hello')).toEqual(delivered)
expect(await app.performAction(element('42.12345'), 'AXWindow.AXPosition.Set', { x: 10, y: 20 })).toEqual(delivered)
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'performAction', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345', action: 'AXPress' } },
{ method: 'performAction', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345', action: 'AXPress', argument: null } },
{ method: 'performAction', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345', action: 'AXTextArea.AXValue.Set', argument: 'hello' } },
{ method: 'performAction', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345', action: 'AXWindow.AXPosition.Set', argument: { x: 10, y: 20 } } },
])
expect(f.sink.text).not.toHaveBeenCalled()
})
it('rejects an empty action before dispatch', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockClear()
await expect(app.performAction(element('0'), '')).rejects.toThrow('performAction action')
expect(f.dispatch).not.toHaveBeenCalled()
})
})
describe('application and window observations', () => {
it('keeps the latest element id scope separate for an app and each selected window', async () => {
const f = fixture()
@@ -94,6 +94,25 @@ it('admits an element action query only with a retained snapshot identity', () =
]) expect(Request.safeParse({ method: 'listActions', params: invalid }).success).toBe(false)
})
it.each([
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXPress' },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXPress', argument: null },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXTextArea.AXValue.Set', argument: 'hello' },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXTextArea.AXSelectedTextRange.Set', argument: { location: 0, length: 3 } },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXWindow.AXPosition.Set', argument: { x: 10, y: 20 } },
])('admits performAction $action with its direct argument', (params) => {
expect(Request.parse({ method: 'performAction', params })).toEqual({ method: 'performAction', params })
})
it('requires a retained snapshot and a nonempty action for performAction', () => {
for (const invalid of [
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0' },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: '' },
{ appHandle: 'app', elementId: '0', action: 'AXPress' },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXPress', extra: true },
]) expect(Request.safeParse({ method: 'performAction', params: invalid }).success).toBe(false)
})
it.each(['click', 'hover', 'scroll', 'drag'])('validates 0–999 integer coordinates for %s', (method) => {
const params = { appHandle: 'app', windowId: 7, relative: true,
...method === 'drag' ? { fromX: 0, fromY: 0, toX: 999, toY: 999 } : { x: 0, y: 999 },
@@ -52,13 +52,21 @@ it('binds all observation and keyboard operations to a selected window and suppr
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'listActions', params: {
windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345',
} })
await window.performAction(element('42.12345'), 'AXTextArea.AXValue.Set', 'hello')
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'performAction', params: {
windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345',
action: 'AXTextArea.AXValue.Set', argument: 'hello',
} })
})
it('accepts an element action query only for a selected window snapshot', () => {
it('accepts an element action query or execution only for a selected window snapshot', () => {
const params = { windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345' }
expect(WindowRequest.parse({ method: 'listActions', params })).toEqual({ method: 'listActions', params })
expect(WindowRequest.safeParse({ method: 'listActions', params: {
windowHandle: 'window-7', elementId: '42.12345',
} }).success).toBe(false)
const action = { ...params, action: 'AXTextArea.AXValue.Set', argument: 'hello' }
expect(WindowRequest.parse({ method: 'performAction', params: action })).toEqual({ method: 'performAction', params: action })
expect(WindowRequest.safeParse({ method: 'performAction', params: { ...params, action: '' } }).success).toBe(false)
})
it('launches a discovered entry without inventing a selected window or waiting for one', async () => {
const f = fixture()
@@ -328,6 +328,52 @@
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"method": {
"type": "string",
"const": "performAction"
},
"params": {
"type": "object",
"properties": {
"appHandle": {
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
"minLength": 1
},
"elementId": {
"description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot.",
"type": "string",
"minLength": 1
},
"action": {
"type": "string",
"minLength": 1
},
"argument": {
"$ref": "#/$defs/__schema0"
}
},
"required": [
"appHandle",
"snapshotId",
"elementId",
"action"
],
"additionalProperties": false
}
},
"required": [
"method",
"params"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
@@ -1258,6 +1304,39 @@
"additionalProperties": false
}
],
"$defs": {
"__schema0": {
"anyOf": [
{
"type": "string"
},
{
"type": "number"
},
{
"type": "boolean"
},
{
"type": "null"
},
{
"type": "array",
"items": {
"$ref": "#/$defs/__schema0"
}
},
{
"type": "object",
"propertyNames": {
"type": "string"
},
"additionalProperties": {
"$ref": "#/$defs/__schema0"
}
}
]
}
},
"type": "object"
}
},