From b0d175ad9d20934b8a2b5aade72e4bc361ba8b59 Mon Sep 17 00:00:00 2001 From: Huo Yaoyuan Date: Mon, 2 Nov 2026 13:16:51 +0800 Subject: [PATCH] feat(cua): list element actions and writable attributes --- .../native/Sources/CuaNative/AXAccess.swift | 27 ++++++++++++ .../native/Sources/CuaNative/AXWorker.swift | 2 + .../CuaNative/NativeProtocol.generated.swift | 3 +- .../cua-macos/native/protocol.schema.json | 42 +++++++++++++++++++ .../native/tests/swift/ListActionsTests.swift | 33 +++++++++++++++ packages/experimental/cua/src/api.ts | 5 ++- packages/experimental/cua/src/protocol.ts | 3 ++ packages/experimental/cua/src/types.ts | 6 +++ .../experimental/cua/src/window-protocol.ts | 1 + packages/experimental/cua/tests/api.spec.ts | 16 +++++++ .../experimental/cua/tests/protocol.spec.ts | 11 +++++ .../experimental/cua/tests/window-api.spec.ts | 12 ++++++ .../cua-native/tool-schemas.expected.json | 38 +++++++++++++++++ 13 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 packages/experimental/cua-macos/native/tests/swift/ListActionsTests.swift diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift index 4bf1d19d4e..1e8cb906b0 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift @@ -157,6 +157,33 @@ final class AXAccess { guard let result = names as? [String] else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility actions are not strings") } return result } + func listActions(_ element: AXUIElement) throws -> [String] { + let advertised = try actions(element) + let attributes = try writableAttributes(element) + guard !attributes.isEmpty else { return advertised } + let role = try string(element, "AXRole") + return try Self.listActionNames(actions: advertised, role: role, writableAttributes: attributes) + } + static func listActionNames(actions: [String], role: String?, writableAttributes: [String]) throws -> [String] { + guard !writableAttributes.isEmpty else { return actions } + guard let role, !role.isEmpty else { + throw NativeFailure(code: "NATIVE_ERROR", message: "Cannot name attribute-setting operations without an accessibility role") + } + return actions + writableAttributes.map { "\(role).\($0).Set" } + } + func writableAttributes(_ element: AXUIElement) throws -> [String] { + try prepare(element) + var names: CFArray? + let error = AXUIElementCopyAttributeNames(element, &names) + if absent(error) { return [] } + guard error == .success else { throw failure(error, "read attribute names") } + guard let names else { return [] } + guard let result = names as? [String] else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility attributes are not strings") } + return try Self.writableAttributeNames(result) { try writable(element, $0) } + } + static func writableAttributeNames(_ names: [String], isWritable: (String) throws -> Bool?) rethrows -> [String] { + try names.filter { try isWritable($0) == true } + } func writable(_ element: AXUIElement, _ name: String) throws -> Bool? { try prepare(element) var result: DarwinBoolean = false diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index 496861b047..fc0779a392 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -300,6 +300,8 @@ actor AXWorker { let window = try coordinateWindow(identity, params.windowID(), trace: trace) trace?.mark("geometry.before.end") return .capture(CaptureTarget(appHandle: handle, process: identity, window: window)) + case "listActions": + return .result(.array(try ax().listActions(element(handle, params)).map { .string($0) })) case "paste": return .paste(PastePlan(handle: handle, text: try params.string("text"), format: try params.optionalString("format") ?? "text", maxBytes: try params.integer("maxClipboardBytes", in: 1024...Int.max), diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift index d0978e7f90..a170b4cf14 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift @@ -1,6 +1,6 @@ // Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit. enum NativeProtocol { - static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listApps", "listWindows", "paste", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"] + static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listActions", "listApps", "listWindows", "paste", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"] static let fields: [String: Set] = [ "activate": ["appHandle"], "applicationIcons": ["paths"], @@ -12,6 +12,7 @@ enum NativeProtocol { "getState": ["appHandle", "details", "includeMenus", "maxDepth", "maxElements", "windowId"], "hover": ["appHandle", "elementId", "snapshotId", "windowId", "x", "y"], "launchApp": ["appHandle"], + "listActions": ["appHandle", "elementId", "snapshotId"], "listApps": [], "listWindows": ["appHandle"], "openPermissionSettings": ["permission"], diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index 08362ee6f6..578b140385 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -273,6 +273,48 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "method": { + "type": "string", + "const": "listActions" + }, + "params": { + "type": "object", + "properties": { + "appHandle": { + "type": "string", + "minLength": 1 + }, + "snapshotId": { + "type": "string", + "minLength": 1 + }, + "elementId": { + "type": "string", + "minLength": 1, + "description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot." + } + }, + "required": [ + "appHandle", + "snapshotId", + "elementId" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "method", + "params" + ], + "additionalProperties": false + }, { "type": "object", "properties": { diff --git a/packages/experimental/cua-macos/native/tests/swift/ListActionsTests.swift b/packages/experimental/cua-macos/native/tests/swift/ListActionsTests.swift new file mode 100644 index 0000000000..81cbae9166 --- /dev/null +++ b/packages/experimental/cua-macos/native/tests/swift/ListActionsTests.swift @@ -0,0 +1,33 @@ +// Action listing includes attribute-setting operations. +import Testing +@testable import CuaNative + +@Test func appendsAttributeSettingActionsAfterNativeActions() throws { + #expect(try AXAccess.listActionNames(actions: ["AXPress"], role: "AXTextArea", + writableAttributes: ["AXValue", "AXSelectedTextRange"]) + == ["AXPress", "AXTextArea.AXValue.Set", "AXTextArea.AXSelectedTextRange.Set"]) +} + +@Test func omitsAttributeSettingActionsWithoutWritableAttributes() throws { + #expect(try AXAccess.listActionNames(actions: ["AXPress"], role: nil, writableAttributes: []) == ["AXPress"]) +} + +@Test func rejectsWritableAttributesWithoutAnObservedRole() { + let roles: [String?] = [nil, ""] + for role in roles { + do { + _ = try AXAccess.listActionNames(actions: [], role: role, writableAttributes: ["AXValue"]) + Issue.record("Attribute-setting names were composed without a role") + } catch let error as NativeFailure { + #expect(error.code == "NATIVE_ERROR") + } catch { Issue.record("Unexpected error: \(error)") } + } +} + +@Test func keepsOnlyAttributesReportedAsWritable() throws { + #expect(try AXAccess.writableAttributeNames(["AXValue", "AXFocused", "AXTitle"]) { name in + if name == "AXValue" { return true } + if name == "AXFocused" { return nil } + return false + } == ["AXValue"]) +} diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index 86caf1a3c8..7e5b9c9889 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -2,7 +2,7 @@ import { Buffer } from 'node:buffer' import type { z } from 'zod' import type { JsonValue } from '@deepseek-ai/dsh-util-values' -import { ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts' +import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts' import type { CuaApi, CuaApp, CuaAppHandle, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts' export type * from './types.ts' @@ -189,6 +189,9 @@ export function createCuaSurface(dispatch: CuaDispatch, sink: CuaObservationSink const state = await result.getAXState(options) return { state, screenshot: await result.getScreenshot(options) } }, + async listActions(elementId) { + return ActionNamesResult.parse(await call('listActions', { ...snapshotParams(), elementId })) + }, async hover(target, options) { const relative = options?.relative === undefined ? {} : { relative: options.relative } if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...snapshotParams(), elementId: target, ...relative })) diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 246c40d3f9..a304ae9de3 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -44,6 +44,7 @@ export const Request = z.discriminatedUnion('method', [ includeMenus: z.boolean().optional().describe('Expand menu contents; omitted or false shows menu-bar entries only.'), details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'), }), + operation('listActions', snapshot), z.object({ method: z.literal('hover'), params: z.union([ z.object(snapshot).strict(), z.object(absoluteCoordinate).strict(), z.object(relativeCoordinate).strict(), ]) }).strict(), @@ -112,6 +113,8 @@ export const StateResult = z.object({ snapshotId: z.string().min(1).transform(value => value as CuaSnapshotId), tree: z.string(), windows: WindowsResult, truncated: z.boolean(), }) +/** Provider-defined action names for one element, including attribute-setting operations; an element may advertise none. */ +export const ActionNamesResult = z.array(z.string()) /** Original native PNG and platform window geometry before Host image preparation. */ export const NativeScreenshotResult = z.object({ mimeType: z.literal('image/png'), data: imageBase64, pixelWidth: id, pixelHeight: id, windowId: id, diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index fa7bb8dbde..5045a64478 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -117,6 +117,12 @@ export interface CuaSurface { * @returns both non-atomic observations. */ getAXStateAndScreenshot(options?: CuaAxOptions): Promise + /** + * List the actions currently advertised by an element, including attribute-setting operations. + * @param elementId - Element id from this surface's latest observation. + * @returns Provider-defined action names; empty when the element advertises none. + */ + listActions(elementId: CuaElementId): Promise /** * Reveal advertised AX alternate UI, otherwise move the system cursor without pressing a button. * The system-pointer route requires the app to be foreground and the point unobscured; never activates or raises it. diff --git a/packages/experimental/cua/src/window-protocol.ts b/packages/experimental/cua/src/window-protocol.ts index 9f63219b05..fd54561ee1 100644 --- a/packages/experimental/cua/src/window-protocol.ts +++ b/packages/experimental/cua/src/window-protocol.ts @@ -25,6 +25,7 @@ export const WindowRequest = z.discriminatedUnion('method', [ operation('findApps', { query: z.string().min(1).regex(/\S/) }), operation('launch', { target: z.string().min(1) }), operation('getState', { ...handle, includeMenus: z.boolean().optional(), details: z.boolean().optional() }), + operation('listActions', snapshot), operation('activate', handle), operation('show', handle), operation('screenshot', handle), operation('typeText', { ...handle, text: z.string() }), operation('paste', { ...handle, text: z.string(), format: z.enum(['text', 'html']).optional() }), diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index 125f220e19..cbf4035151 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -37,6 +37,7 @@ function fixture(withSink = true) { windows: params.windowId === undefined ? windows : windows.filter(window => window.windowId === params.windowId), } case 'screenshot': return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: params.windowId, bounds, coordinateSpace: 'screenshot-pixels' } + case 'listActions': return ['AXPress', 'AXShowMenu', 'AXTextArea.AXValue.Set'] default: return delivered } }) @@ -232,6 +233,21 @@ describe('text selections', () => { }) }) +describe('element action queries', () => { + it('returns raw provider action names including attribute-setting operations without reading new state', async () => { + const f = fixture() + const app = await f.api.getApp('Editor') + f.dispatch.mockClear() + expect(await app.listActions(element('42.12345'))).toEqual(['AXPress', 'AXShowMenu', 'AXTextArea.AXValue.Set']) + expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([ + { method: 'listActions', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345' } }, + ]) + await expect(app.listActions(element(''))).rejects.toThrow() + expect(f.dispatch).toHaveBeenCalledTimes(1) + expect(f.sink.text).not.toHaveBeenCalled() + }) +}) + describe('application and window observations', () => { it('keeps the latest element id scope separate for an app and each selected window', async () => { const f = fixture() diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index 2738587f73..8f05dd6721 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -83,6 +83,17 @@ it.each(['0', '42.12345', 'runtime:7/child', 'e12'])('admits an opaque element i expect(Request.safeParse({ method: 'click', params: { ...params, elementId: '' } }).success).toBe(false) }) +it('admits an element action query only with a retained snapshot identity', () => { + const params = { appHandle: 'app', snapshotId: 'snapshot', elementId: '42.12345' } + expect(Request.parse({ method: 'listActions', params })).toEqual({ method: 'listActions', params }) + for (const invalid of [ + { appHandle: 'app', elementId: '0' }, + { appHandle: 'app', snapshotId: 'snapshot' }, + { appHandle: 'app', snapshotId: 'snapshot', elementId: '' }, + { appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXPress' }, + ]) expect(Request.safeParse({ method: 'listActions', params: invalid }).success).toBe(false) +}) + it.each(['click', 'hover', 'scroll', 'drag'])('validates 0–999 integer coordinates for %s', (method) => { const params = { appHandle: 'app', windowId: 7, relative: true, ...method === 'drag' ? { fromX: 0, fromY: 0, toX: 999, toY: 999 } : { x: 0, y: 999 }, diff --git a/packages/experimental/cua/tests/window-api.spec.ts b/packages/experimental/cua/tests/window-api.spec.ts index 2ad38196eb..fc2fc4e761 100644 --- a/packages/experimental/cua/tests/window-api.spec.ts +++ b/packages/experimental/cua/tests/window-api.spec.ts @@ -19,6 +19,7 @@ function fixture() { case 'getWindows': return request.params.ids.map(windowId => ({ windowId, pid: 10, windowHandle: 'window-7' })) case 'getState': return { windowHandle: 'window-7', windowId: 7, pid: 10, snapshotId: 'state-1', tree: 'AX window', windows: [metadata], truncated: false } case 'findApps': return [{ id: 'shell-entry', displayName: 'Settings', kind: 'shell' }] + case 'listActions': return ['AXPress'] default: return { dispatched: true, verified: false } } }) @@ -47,6 +48,17 @@ it('binds all observation and keyboard operations to a selected window and suppr await window.pressKey('ctrl+a') expect(f.dispatch.mock.calls.slice(2).every(([r]) => 'windowHandle' in r.params && r.params.windowHandle === 'window-7')).toBe(true) expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { windowHandle: 'window-7', key: 'a', modifiers: ['control'] } }) + expect(await window.listActions(element('42.12345'))).toEqual(['AXPress']) + expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'listActions', params: { + windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345', + } }) +}) +it('accepts an element action query only for a selected window snapshot', () => { + const params = { windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345' } + expect(WindowRequest.parse({ method: 'listActions', params })).toEqual({ method: 'listActions', params }) + expect(WindowRequest.safeParse({ method: 'listActions', params: { + windowHandle: 'window-7', elementId: '42.12345', + } }).success).toBe(false) }) it('launches a discovered entry without inventing a selected window or waiting for one', async () => { const f = fixture() diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index fda8bbed4d..f74f96338d 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -290,6 +290,44 @@ ], "additionalProperties": false }, + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "listActions" + }, + "params": { + "type": "object", + "properties": { + "appHandle": { + "type": "string", + "minLength": 1 + }, + "snapshotId": { + "type": "string", + "minLength": 1 + }, + "elementId": { + "description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot.", + "type": "string", + "minLength": 1 + } + }, + "required": [ + "appHandle", + "snapshotId", + "elementId" + ], + "additionalProperties": false + } + }, + "required": [ + "method", + "params" + ], + "additionalProperties": false + }, { "type": "object", "properties": {