feat(cua): list element actions and writable attributes
This commit is contained in:
1 parent
4b73aab79a
commit
b0d175ad9d
13 files changed
+197
-2
No files matched your search
@@ -157,6 +157,33 @@ final class AXAccess {
|
||||
guard let result = names as? [String] else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility actions are not strings") }
|
||||
return result
|
||||
}
|
||||
func listActions(_ element: AXUIElement) throws -> [String] {
|
||||
let advertised = try actions(element)
|
||||
let attributes = try writableAttributes(element)
|
||||
guard !attributes.isEmpty else { return advertised }
|
||||
let role = try string(element, "AXRole")
|
||||
return try Self.listActionNames(actions: advertised, role: role, writableAttributes: attributes)
|
||||
}
|
||||
static func listActionNames(actions: [String], role: String?, writableAttributes: [String]) throws -> [String] {
|
||||
guard !writableAttributes.isEmpty else { return actions }
|
||||
guard let role, !role.isEmpty else {
|
||||
throw NativeFailure(code: "NATIVE_ERROR", message: "Cannot name attribute-setting operations without an accessibility role")
|
||||
}
|
||||
return actions + writableAttributes.map { "\(role).\($0).Set" }
|
||||
}
|
||||
func writableAttributes(_ element: AXUIElement) throws -> [String] {
|
||||
try prepare(element)
|
||||
var names: CFArray?
|
||||
let error = AXUIElementCopyAttributeNames(element, &names)
|
||||
if absent(error) { return [] }
|
||||
guard error == .success else { throw failure(error, "read attribute names") }
|
||||
guard let names else { return [] }
|
||||
guard let result = names as? [String] else { throw NativeFailure(code: "NATIVE_ERROR", message: "Accessibility attributes are not strings") }
|
||||
return try Self.writableAttributeNames(result) { try writable(element, $0) }
|
||||
}
|
||||
static func writableAttributeNames(_ names: [String], isWritable: (String) throws -> Bool?) rethrows -> [String] {
|
||||
try names.filter { try isWritable($0) == true }
|
||||
}
|
||||
func writable(_ element: AXUIElement, _ name: String) throws -> Bool? {
|
||||
try prepare(element)
|
||||
var result: DarwinBoolean = false
|
||||
|
||||
@@ -300,6 +300,8 @@ actor AXWorker {
|
||||
let window = try coordinateWindow(identity, params.windowID(), trace: trace)
|
||||
trace?.mark("geometry.before.end")
|
||||
return .capture(CaptureTarget(appHandle: handle, process: identity, window: window))
|
||||
case "listActions":
|
||||
return .result(.array(try ax().listActions(element(handle, params)).map { .string($0) }))
|
||||
case "paste":
|
||||
return .paste(PastePlan(handle: handle, text: try params.string("text"), format: try params.optionalString("format") ?? "text",
|
||||
maxBytes: try params.integer("maxClipboardBytes", in: 1024...Int.max),
|
||||
|
||||
+2
-1
@@ -1,6 +1,6 @@
|
||||
// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.
|
||||
enum NativeProtocol {
|
||||
static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listApps", "listWindows", "paste", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"]
|
||||
static let modelMethods: [String] = ["activate", "click", "drag", "findApps", "getApp", "getCapabilities", "getState", "hover", "listActions", "listApps", "listWindows", "paste", "pressKey", "screenshot", "scroll", "selectText", "setValue", "show", "typeText"]
|
||||
static let fields: [String: Set<String>] = [
|
||||
"activate": ["appHandle"],
|
||||
"applicationIcons": ["paths"],
|
||||
@@ -12,6 +12,7 @@ enum NativeProtocol {
|
||||
"getState": ["appHandle", "details", "includeMenus", "maxDepth", "maxElements", "windowId"],
|
||||
"hover": ["appHandle", "elementId", "snapshotId", "windowId", "x", "y"],
|
||||
"launchApp": ["appHandle"],
|
||||
"listActions": ["appHandle", "elementId", "snapshotId"],
|
||||
"listApps": [],
|
||||
"listWindows": ["appHandle"],
|
||||
"openPermissionSettings": ["permission"],
|
||||
|
||||
@@ -273,6 +273,48 @@
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"method": {
|
||||
"type": "string",
|
||||
"const": "listActions"
|
||||
},
|
||||
"params": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"elementId": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"snapshotId",
|
||||
"elementId"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"method",
|
||||
"params"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Action listing includes attribute-setting operations.
|
||||
import Testing
|
||||
@testable import CuaNative
|
||||
|
||||
@Test func appendsAttributeSettingActionsAfterNativeActions() throws {
|
||||
#expect(try AXAccess.listActionNames(actions: ["AXPress"], role: "AXTextArea",
|
||||
writableAttributes: ["AXValue", "AXSelectedTextRange"])
|
||||
== ["AXPress", "AXTextArea.AXValue.Set", "AXTextArea.AXSelectedTextRange.Set"])
|
||||
}
|
||||
|
||||
@Test func omitsAttributeSettingActionsWithoutWritableAttributes() throws {
|
||||
#expect(try AXAccess.listActionNames(actions: ["AXPress"], role: nil, writableAttributes: []) == ["AXPress"])
|
||||
}
|
||||
|
||||
@Test func rejectsWritableAttributesWithoutAnObservedRole() {
|
||||
let roles: [String?] = [nil, ""]
|
||||
for role in roles {
|
||||
do {
|
||||
_ = try AXAccess.listActionNames(actions: [], role: role, writableAttributes: ["AXValue"])
|
||||
Issue.record("Attribute-setting names were composed without a role")
|
||||
} catch let error as NativeFailure {
|
||||
#expect(error.code == "NATIVE_ERROR")
|
||||
} catch { Issue.record("Unexpected error: \(error)") }
|
||||
}
|
||||
}
|
||||
|
||||
@Test func keepsOnlyAttributesReportedAsWritable() throws {
|
||||
#expect(try AXAccess.writableAttributeNames(["AXValue", "AXFocused", "AXTitle"]) { name in
|
||||
if name == "AXValue" { return true }
|
||||
if name == "AXFocused" { return nil }
|
||||
return false
|
||||
} == ["AXValue"])
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
import { Buffer } from 'node:buffer'
|
||||
import type { z } from 'zod'
|
||||
import type { JsonValue } from '@deepseek-ai/dsh-util-values'
|
||||
import { ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts'
|
||||
import { ActionNamesResult, ActionResult, AppResult, AppsResult, CapabilitiesResult, FoundAppsResult, HoverResult, Request, ScreenshotResult, ShowResult, StateResult, WindowsResult } from './protocol.ts'
|
||||
import type { CuaApi, CuaApp, CuaAppHandle, CuaDispatch, CuaObservationOptions, CuaObservationSink, CuaOperation, CuaScreenshot, CuaSurface, CuaWindow } from './types.ts'
|
||||
export type * from './types.ts'
|
||||
|
||||
@@ -189,6 +189,9 @@ export function createCuaSurface(dispatch: CuaDispatch, sink: CuaObservationSink
|
||||
const state = await result.getAXState(options)
|
||||
return { state, screenshot: await result.getScreenshot(options) }
|
||||
},
|
||||
async listActions(elementId) {
|
||||
return ActionNamesResult.parse(await call('listActions', { ...snapshotParams(), elementId }))
|
||||
},
|
||||
async hover(target, options) {
|
||||
const relative = options?.relative === undefined ? {} : { relative: options.relative }
|
||||
if (typeof target === 'string') return HoverResult.parse(await call('hover', { ...snapshotParams(), elementId: target, ...relative }))
|
||||
|
||||
@@ -44,6 +44,7 @@ export const Request = z.discriminatedUnion('method', [
|
||||
includeMenus: z.boolean().optional().describe('Expand menu contents; omitted or false shows menu-bar entries only.'),
|
||||
details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'),
|
||||
}),
|
||||
operation('listActions', snapshot),
|
||||
z.object({ method: z.literal('hover'), params: z.union([
|
||||
z.object(snapshot).strict(), z.object(absoluteCoordinate).strict(), z.object(relativeCoordinate).strict(),
|
||||
]) }).strict(),
|
||||
@@ -112,6 +113,8 @@ export const StateResult = z.object({
|
||||
snapshotId: z.string().min(1).transform(value => value as CuaSnapshotId),
|
||||
tree: z.string(), windows: WindowsResult, truncated: z.boolean(),
|
||||
})
|
||||
/** Provider-defined action names for one element, including attribute-setting operations; an element may advertise none. */
|
||||
export const ActionNamesResult = z.array(z.string())
|
||||
/** Original native PNG and platform window geometry before Host image preparation. */
|
||||
export const NativeScreenshotResult = z.object({
|
||||
mimeType: z.literal('image/png'), data: imageBase64, pixelWidth: id, pixelHeight: id, windowId: id,
|
||||
|
||||
@@ -117,6 +117,12 @@ export interface CuaSurface {
|
||||
* @returns both non-atomic observations.
|
||||
*/
|
||||
getAXStateAndScreenshot(options?: CuaAxOptions): Promise<CuaObservation>
|
||||
/**
|
||||
* List the actions currently advertised by an element, including attribute-setting operations.
|
||||
* @param elementId - Element id from this surface's latest observation.
|
||||
* @returns Provider-defined action names; empty when the element advertises none.
|
||||
*/
|
||||
listActions(elementId: CuaElementId): Promise<string[]>
|
||||
/**
|
||||
* Reveal advertised AX alternate UI, otherwise move the system cursor without pressing a button.
|
||||
* The system-pointer route requires the app to be foreground and the point unobscured; never activates or raises it.
|
||||
|
||||
@@ -25,6 +25,7 @@ export const WindowRequest = z.discriminatedUnion('method', [
|
||||
operation('findApps', { query: z.string().min(1).regex(/\S/) }),
|
||||
operation('launch', { target: z.string().min(1) }),
|
||||
operation('getState', { ...handle, includeMenus: z.boolean().optional(), details: z.boolean().optional() }),
|
||||
operation('listActions', snapshot),
|
||||
operation('activate', handle), operation('show', handle), operation('screenshot', handle),
|
||||
operation('typeText', { ...handle, text: z.string() }),
|
||||
operation('paste', { ...handle, text: z.string(), format: z.enum(['text', 'html']).optional() }),
|
||||
|
||||
@@ -37,6 +37,7 @@ function fixture(withSink = true) {
|
||||
windows: params.windowId === undefined ? windows : windows.filter(window => window.windowId === params.windowId),
|
||||
}
|
||||
case 'screenshot': return { mimeType: 'image/png', data: PNG, pixelWidth: 1, pixelHeight: 1, windowId: params.windowId, bounds, coordinateSpace: 'screenshot-pixels' }
|
||||
case 'listActions': return ['AXPress', 'AXShowMenu', 'AXTextArea.AXValue.Set']
|
||||
default: return delivered
|
||||
}
|
||||
})
|
||||
@@ -232,6 +233,21 @@ describe('text selections', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('element action queries', () => {
|
||||
it('returns raw provider action names including attribute-setting operations without reading new state', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.api.getApp('Editor')
|
||||
f.dispatch.mockClear()
|
||||
expect(await app.listActions(element('42.12345'))).toEqual(['AXPress', 'AXShowMenu', 'AXTextArea.AXValue.Set'])
|
||||
expect(f.dispatch.mock.calls.map(([operation]) => operation)).toEqual([
|
||||
{ method: 'listActions', params: { appHandle: 'app-17', snapshotId: 'snapshot-1', elementId: '42.12345' } },
|
||||
])
|
||||
await expect(app.listActions(element(''))).rejects.toThrow()
|
||||
expect(f.dispatch).toHaveBeenCalledTimes(1)
|
||||
expect(f.sink.text).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('application and window observations', () => {
|
||||
it('keeps the latest element id scope separate for an app and each selected window', async () => {
|
||||
const f = fixture()
|
||||
|
||||
@@ -83,6 +83,17 @@ it.each(['0', '42.12345', 'runtime:7/child', 'e12'])('admits an opaque element i
|
||||
expect(Request.safeParse({ method: 'click', params: { ...params, elementId: '' } }).success).toBe(false)
|
||||
})
|
||||
|
||||
it('admits an element action query only with a retained snapshot identity', () => {
|
||||
const params = { appHandle: 'app', snapshotId: 'snapshot', elementId: '42.12345' }
|
||||
expect(Request.parse({ method: 'listActions', params })).toEqual({ method: 'listActions', params })
|
||||
for (const invalid of [
|
||||
{ appHandle: 'app', elementId: '0' },
|
||||
{ appHandle: 'app', snapshotId: 'snapshot' },
|
||||
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '' },
|
||||
{ appHandle: 'app', snapshotId: 'snapshot', elementId: '0', action: 'AXPress' },
|
||||
]) expect(Request.safeParse({ method: 'listActions', params: invalid }).success).toBe(false)
|
||||
})
|
||||
|
||||
it.each(['click', 'hover', 'scroll', 'drag'])('validates 0–999 integer coordinates for %s', (method) => {
|
||||
const params = { appHandle: 'app', windowId: 7, relative: true,
|
||||
...method === 'drag' ? { fromX: 0, fromY: 0, toX: 999, toY: 999 } : { x: 0, y: 999 },
|
||||
|
||||
@@ -19,6 +19,7 @@ function fixture() {
|
||||
case 'getWindows': return request.params.ids.map(windowId => ({ windowId, pid: 10, windowHandle: 'window-7' }))
|
||||
case 'getState': return { windowHandle: 'window-7', windowId: 7, pid: 10, snapshotId: 'state-1', tree: 'AX window', windows: [metadata], truncated: false }
|
||||
case 'findApps': return [{ id: 'shell-entry', displayName: 'Settings', kind: 'shell' }]
|
||||
case 'listActions': return ['AXPress']
|
||||
default: return { dispatched: true, verified: false }
|
||||
}
|
||||
})
|
||||
@@ -47,6 +48,17 @@ it('binds all observation and keyboard operations to a selected window and suppr
|
||||
await window.pressKey('ctrl+a')
|
||||
expect(f.dispatch.mock.calls.slice(2).every(([r]) => 'windowHandle' in r.params && r.params.windowHandle === 'window-7')).toBe(true)
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'pressKey', params: { windowHandle: 'window-7', key: 'a', modifiers: ['control'] } })
|
||||
expect(await window.listActions(element('42.12345'))).toEqual(['AXPress'])
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ method: 'listActions', params: {
|
||||
windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345',
|
||||
} })
|
||||
})
|
||||
it('accepts an element action query only for a selected window snapshot', () => {
|
||||
const params = { windowHandle: 'window-7', snapshotId: 'state-1', elementId: '42.12345' }
|
||||
expect(WindowRequest.parse({ method: 'listActions', params })).toEqual({ method: 'listActions', params })
|
||||
expect(WindowRequest.safeParse({ method: 'listActions', params: {
|
||||
windowHandle: 'window-7', elementId: '42.12345',
|
||||
} }).success).toBe(false)
|
||||
})
|
||||
it('launches a discovered entry without inventing a selected window or waiting for one', async () => {
|
||||
const f = fixture()
|
||||
|
||||
@@ -290,6 +290,44 @@
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"method": {
|
||||
"type": "string",
|
||||
"const": "listActions"
|
||||
},
|
||||
"params": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"elementId": {
|
||||
"description": "Opaque string defined by the platform provider, valid for at least the lifetime of one snapshot.",
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"snapshotId",
|
||||
"elementId"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"method",
|
||||
"params"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
Reference in new issue
Block a user