feat(cua): list element actions and writable attributes

This commit is contained in:
Huo Yaoyuan committed 2026-11-02 13:16:51 +08:00
1 parent 4b73aab79a
commit b0d175ad9d
13 files changed
+197 -2

No files matched your search

@@ -157,6 +157,33 @@ final class AXAccess {
guard let result = names as? [String] else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility actions are not strings") }
return result
}
func listActions(_ element: AXUIElement) throws -> [String] {
let advertised = try actions(element)
let attributes = try writableAttributes(element)
guard !attributes.isEmpty else { return advertised }
let role = try string(element, "AXRole")
return try Self.listActionNames(actions: advertised, role: role, writableAttributes: attributes)
}
static func listActionNames(actions: [String], role: String?, writableAttributes: [String]) throws -> [String] {
guard !writableAttributes.isEmpty else { return actions }
guard let role, !role.isEmpty else {
throw NativeFailure(code: "NATIVE_ERROR", message: "Cannot name attribute-setting operations without an accessibility role")
}
return actions + writableAttributes.map { "\(role).\($0).Set" }
}
func writableAttributes(_ element: AXUIElement) throws -> [String] {
try prepare(element)
var names: CFArray?
let error = AXUIElementCopyAttributeNames(element, &names)
if absent(error) { return [] }
guard error == .success else { throw failure(error, "read attribute names") }
guard let names else { return [] }
guard let result = names as? [String] else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility attributes are not strings") }
return try Self.writableAttributeNames(result) { try writable(element, $0) }
}
static func writableAttributeNames(_ names: [String], isWritable: (String) throws -> Bool?) rethrows -> [String] {
try names.filter { try isWritable($0) == true }
}
func writable(_ element: AXUIElement, _ name: String) throws -> Bool? {
try prepare(element)
var result: DarwinBoolean = false
@@ -300,6 +300,8 @@ actor AXWorker {
let window = try coordinateWindow(identity, params.windowID(), trace: trace)
trace?.mark("geometry.before.end")
return .capture(CaptureTarget(appHandle: handle, process: identity, window: window))
case "listActions":
return .result(.array(try ax().listActions(element(handle, params)).map { .string($0) }))
case "paste":
return .paste(PastePlan(handle: handle, text: try params.string("text"), format: try params.optionalString("format") ?? "text",
maxBytes: try params.integer("maxClipboardBytes", in: 1024...Int.max),
@@ -1,6 +1,6 @@
// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.
enum NativeProtocol {
static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listApps", "listWindows", "paste", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"]
static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listActions", "listApps", "listWindows", "paste", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"]
static let fields: [String: Set<String>] = [
"activate": ["appHandle"],
"applicationIcons": ["paths"],
@@ -12,6 +12,7 @@ enum NativeProtocol {
"getState": ["appHandle", "details", "includeMenus", "maxDepth", "maxElements", "windowId"],
"hover": ["appHandle", "elementId", "snapshotId", "windowId", "x", "y"],
"launchApp": ["appHandle"],
"listActions": ["appHandle", "elementId", "snapshotId"],
"listApps": [],
"listWindows": ["appHandle"],
"openPermissionSettings": ["permission"],
@@ -273,6 +273,48 @@
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"id": {
"type": "string"
},
"method": {
"type": "string",
"const": "listActions"
},
"params": {
"type": "object",
"properties": {
"appHandle": {
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
"minLength": 1
},
"elementId": {
"type": "string",
"minLength": 1,
"description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot."
}
},
"required": [
"appHandle",
"snapshotId",
"elementId"
],
"additionalProperties": false
}
},
"required": [
"id",
"method",
"params"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
@@ -0,0 +1,33 @@
// Action listing includes attribute-setting operations.
import Testing
@testable import CuaNative
@Test func appendsAttributeSettingActionsAfterNativeActions() throws {
#expect(try AXAccess.listActionNames(actions: ["AXPress"], role: "AXTextArea",
writableAttributes: ["AXValue", "AXSelectedTextRange"])
== ["AXPress", "AXTextArea.AXValue.Set", "AXTextArea.AXSelectedTextRange.Set"])
}
@Test func omitsAttributeSettingActionsWithoutWritableAttributes() throws {
#expect(try AXAccess.listActionNames(actions: ["AXPress"], role: nil, writableAttributes: []) == ["AXPress"])
}
@Test func rejectsWritableAttributesWithoutAnObservedRole() {
let roles: [String?] = [nil, ""]
for role in roles {
do {
_ = try AXAccess.listActionNames(actions: [], role: role, writableAttributes: ["AXValue"])
Issue.record("Attribute-setting names were composed without a role")
} catch let error as NativeFailure {
#expect(error.code == "NATIVE_ERROR")
} catch { Issue.record("Unexpected error: \(error)") }
}
}
@Test func keepsOnlyAttributesReportedAsWritable() throws {
#expect(try AXAccess.writableAttributeNames(["AXValue", "AXFocused", "AXTitle"]) { name in
if name == "AXValue" { return true }
if name == "AXFocused" { return nil }
return false
} == ["AXValue"])
}
+4 -1
View File
@@ -2,7 +2,7 @@
import { Buffer } from 'node:buffer'
import type { z } from 'zod'
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
import { ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts'
import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts'
import type { CuaApi, CuaApp, CuaAppHandle, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts'
export type * from './types.ts'
@@ -189,6 +189,9 @@ export function createCuaSurface(dispatch: CuaDispatch, sink: CuaObservationSink
const state = await result.getAXState(options)
return { state, screenshot: await result.getScreenshot(options) }
},
async listActions(elementId) {
return ActionNamesResult.parse(await call('listActions', { ...snapshotParams(), elementId }))
},
async hover(target, options) {
const relative = options?.relative === undefined ? {} : { relative: options.relative }
if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...snapshotParams(), elementId: target, ...relative }))
@@ -44,6 +44,7 @@ export const Request = z.discriminatedUnion('method', [
includeMenus: z.boolean().optional().describe('Expand menu contents; omitted or false shows menu-bar entries only.'),
details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'),
}),
operation('listActions', snapshot),
z.object({ method: z.literal('hover'), params: z.union([
z.object(snapshot).strict(), z.object(absoluteCoordinate).strict(), z.object(relativeCoordinate).strict(),
]) }).strict(),
@@ -112,6 +113,8 @@ export const StateResult = z.object({
snapshotId: z.string().min(1).transform(value => value as CuaSnapshotId),
tree: z.string(), windows: WindowsResult, truncated: z.boolean(),
})
/** Provider-defined action names for one element, including attribute-setting operations; an element may advertise none. */
export const ActionNamesResult = z.array(z.string())
/** Original native PNG and platform window geometry before Host image preparation. */
export const NativeScreenshotResult = z.object({
mimeType: z.literal('image/png'), data: imageBase64, pixelWidth: id, pixelHeight: id, windowId: id,
+6
View File
@@ -117,6 +117,12 @@ export interface CuaSurface {
* @returns both non-atomic observations.
*/
getAXStateAndScreenshot(options?: CuaAxOptions): Promise<CuaObservation>
/**
* List the actions currently advertised by an element, including attribute-setting operations.
* @param elementId - Element id from this surface's latest observation.
* @returns Provider-defined action names; empty when the element advertises none.
*/
listActions(elementId: CuaElementId): Promise<string[]>
/**
* Reveal advertised AX alternate UI, otherwise move the system cursor without pressing a button.
* The system-pointer route requires the app to be foreground and the point unobscured; never activates or raises it.
@@ -25,6 +25,7 @@ export const WindowRequest = z.discriminatedUnion('method', [
operation('findApps', { query: z.string().min(1).regex(/\S/) }),
operation('launch', { target: z.string().min(1) }),
operation('getState', { ...handle, includeMenus: z.boolean().optional(), details: z.boolean().optional() }),
operation('listActions', snapshot),
operation('activate', handle), operation('show', handle), operation('screenshot', handle),
operation('typeText', { ...handle, text: z.string() }),
operation('paste', { ...handle, text: z.string(), format: z.enum(['text', 'html']).optional() }),
@@ -37,6 +37,7 @@ function fixture(withSink = true) {
windows: params.windowId === undefined ? windows : windows.filter(window => window.windowId === params.windowId),
}
case 'screenshot': return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: params.windowId, bounds, coordinateSpace: 'screenshot-pixels' }
case 'listActions': return ['AXPress', 'AXShowMenu', 'AXTextArea.AXValue.Set']
default: return delivered
}
})
@@ -232,6 +233,21 @@ describe('text selections', () => {
})
})
describe('element action queries', () => {
it('returns raw provider action names including attribute-setting operations without reading new state', async () => {
const f = fixture()
const app = await f.api.getApp('Editor')
f.dispatch.mockClear()
expect(await app.listActions(element('42.12345'))).toEqual(['AXPress', 'AXShowMenu', 'AXTextArea.AXValue.Set'])
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
{ method: 'listActions', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345' } },
])
await expect(app.listActions(element(''))).rejects.toThrow()
expect(f.dispatch).toHaveBeenCalledTimes(1)
expect(f.sink.text).not.toHaveBeenCalled()
})
})
describe('application and window observations', () => {
it('keeps the latest element id scope separate for an app and each selected window', async () => {
const f = fixture()
@@ -83,6 +83,17 @@ it.each(['0', '42.12345', 'runtime:7/child', 'e12'])('admits an opaque element i
expect(Request.safeParse({ method: 'click', params: { ...params, elementId: '' } }).success).toBe(false)
})
it('admits an element action query only with a retained snapshot identity', () => {
const params = { appHandle: 'app', snapshotId: 'snapshot', elementId: '42.12345' }
expect(Request.parse({ method: 'listActions', params })).toEqual({ method: 'listActions', params })
for (const invalid of [
{ appHandle: 'app', elementId: '0' },
{ appHandle: 'app', snapshotId: 'snapshot' },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '' },
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXPress' },
]) expect(Request.safeParse({ method: 'listActions', params: invalid }).success).toBe(false)
})
it.each(['click', 'hover', 'scroll', 'drag'])('validates 0–999 integer coordinates for %s', (method) => {
const params = { appHandle: 'app', windowId: 7, relative: true,
...method === 'drag' ? { fromX: 0, fromY: 0, toX: 999, toY: 999 } : { x: 0, y: 999 },
@@ -19,6 +19,7 @@ function fixture() {
case 'getWindows': return request.params.ids.map(windowId => ({ windowId, pid: 10, windowHandle: 'window-7' }))
case 'getState': return { windowHandle: 'window-7', windowId: 7, pid: 10, snapshotId: 'state-1', tree: 'AX window', windows: [metadata], truncated: false }
case 'findApps': return [{ id: 'shell-entry', displayName: 'Settings', kind: 'shell' }]
case 'listActions': return ['AXPress']
default: return { dispatched: true, verified: false }
}
})
@@ -47,6 +48,17 @@ it('binds all observation and keyboard operations to a selected window and suppr
await window.pressKey('ctrl+a')
expect(f.dispatch.mock.calls.slice(2).every(([r]) => 'windowHandle' in r.params && r.params.windowHandle === 'window-7')).toBe(true)
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { windowHandle: 'window-7', key: 'a', modifiers: ['control'] } })
expect(await window.listActions(element('42.12345'))).toEqual(['AXPress'])
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'listActions', params: {
windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345',
} })
})
it('accepts an element action query only for a selected window snapshot', () => {
const params = { windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345' }
expect(WindowRequest.parse({ method: 'listActions', params })).toEqual({ method: 'listActions', params })
expect(WindowRequest.safeParse({ method: 'listActions', params: {
windowHandle: 'window-7', elementId: '42.12345',
} }).success).toBe(false)
})
it('launches a discovered entry without inventing a selected window or waiting for one', async () => {
const f = fixture()
@@ -290,6 +290,44 @@
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"method": {
"type": "string",
"const": "listActions"
},
"params": {
"type": "object",
"properties": {
"appHandle": {
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
"minLength": 1
},
"elementId": {
"description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot.",
"type": "string",
"minLength": 1
}
},
"required": [
"appHandle",
"snapshotId",
"elementId"
],
"additionalProperties": false
}
},
"required": [
"method",
"params"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {