fix(cua): retain other window mappings during scoped observations
This commit is contained in:
creatixchu committed
1 parent
06e671f143
commit
ff1864c6cf
8 files changed
+406
-123
No files matched your search
@@ -17,11 +17,13 @@ export class CuaCoordinates {
|
||||
/**
|
||||
* Invalidate screenshot mappings when a window closes or changes size.
|
||||
* @param appHandle - native application selection.
|
||||
* @param windows - fresh complete window inventory for this application.
|
||||
* @param windows - freshly observed windows.
|
||||
* @param windowId - selected window for a scoped observation; omission means a complete application inventory.
|
||||
*/
|
||||
observe(appHandle: CuaAppHandle, windows: readonly CuaWindowInfo[]): void {
|
||||
observe(appHandle: CuaAppHandle, windows: readonly CuaWindowInfo[], windowId?: number): void {
|
||||
const previous = this.applications.get(appHandle)
|
||||
const current = new Map<number, Frame>()
|
||||
const current = new Map<number, Frame>(windowId === undefined ? [] : previous)
|
||||
if (windowId !== undefined && !windows.some(window => window.windowId === windowId)) current.delete(windowId)
|
||||
for (const window of windows) {
|
||||
const { width, height } = window.bounds
|
||||
const old = previous?.get(window.windowId)
|
||||
|
||||
@@ -266,7 +266,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend)
|
||||
if (method === 'getCapabilities') return { ...z.record(z.string(), z.json()).parse(result), coordinateSpace: 'screenshot-pixels' }
|
||||
if (method === 'getState') {
|
||||
const state = StateResult.parse(result)
|
||||
execution.coordinates.observe(state.appHandle, state.windows)
|
||||
execution.coordinates.observe(state.appHandle, state.windows, state.windowId ?? undefined)
|
||||
return state
|
||||
}
|
||||
if (method === 'listWindows') execution.coordinates.observe(parsedParams.appHandle as CuaAppHandle, WindowsResult.parse(result))
|
||||
|
||||
@@ -73,3 +73,14 @@ it('keeps different application/window mappings independent and leaves element a
|
||||
const request = Request.parse({ method: 'hover', params: { appHandle: handle, snapshotId: 's', elementIndex: 2 } })
|
||||
expect(coordinates.parameters(request)).toEqual(request.params)
|
||||
})
|
||||
|
||||
it('keeps other windows mapped during scoped observations and only removes the scoped closed window', () => {
|
||||
const coordinates = new CuaCoordinates()
|
||||
coordinates.screenshot(handle, 7, window().bounds, { width: 1280, height: 960 })
|
||||
coordinates.observe(handle, [{ ...window(), windowId: 8 }], 8)
|
||||
expect(coordinates.parameters(call('click', { x: 640, y: 480 }))).toMatchObject({ x: 320, y: 240 })
|
||||
coordinates.observe(handle, [], 8)
|
||||
expect(coordinates.parameters(call('click', { x: 640, y: 480 }))).toMatchObject({ x: 320, y: 240 })
|
||||
coordinates.observe(handle, [], 7)
|
||||
expect(() => coordinates.parameters(call('click', { x: 0, y: 0 }))).toThrow('STALE_SNAPSHOT')
|
||||
})
|
||||
@@ -29,7 +29,7 @@ Use the workflow tool ONLY when the user explicitly asks for a workflow or for l
|
||||
|
||||
Start independent delegations with `subagent` or `subagent_fork` together in one assistant message and continue useful work while they run.
|
||||
|
||||
Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are window-local macOS points, which may differ from screenshot pixels. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
|
||||
Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
|
||||
|
||||
Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.
|
||||
|
||||
@@ -54,18 +54,18 @@ App and Window methods:
|
||||
- getAXState(options?: { emit?: boolean, disableDiffing?: boolean, includeMenus?: boolean, details?: boolean }) -> string
|
||||
- getScreenshot(options?: { emit?: boolean }) -> Uint8Array
|
||||
- getAXStateAndScreenshot(options?) -> { state: string, screenshot: Uint8Array }
|
||||
- hover(target: number | [x, y]) -> action result
|
||||
- click(target: number | [x, y], options?: { mouseButton?: "left" | "right" | "middle", clickCount?: 1 | 2, modifiers?: Array<"command" | "control" | "option" | "shift" | "fn"> }) -> action result
|
||||
- hover(target: number | [x, y], options?: { relative?: boolean }) -> { dispatched, verified, hoverRoute, systemPointerMoved }
|
||||
- click(target: number | [x, y], options?: { mouseButton?: "left" | "right" | "middle", clickCount?: 1 | 2, modifiers?: Array<"command" | "control" | "option" | "shift" | "fn">, relative?: boolean }) -> action result
|
||||
- setValue(elementIndex: number, value: string) -> action result
|
||||
- selectText(elementIndex: number, text: string, options?: { prefix?: string, suffix?: string, selectionType?: "text" | "cursor_before" | "cursor_after" }) -> action result
|
||||
- performSecondaryAction(elementIndex: number, action: string) -> action result
|
||||
- scroll({ x, y, deltaLinesX, deltaLinesY }) -> action result
|
||||
- drag(from: [x, y], to: [x, y], options?: { mouseButton?: "left" | "right" | "middle", modifiers?: Array<"command" | "control" | "option" | "shift" | "fn"> }) -> action result
|
||||
- scroll({ x, y, deltaLinesX, deltaLinesY, relative? }) -> action result
|
||||
- drag(from: [x, y], to: [x, y], options?: { mouseButton?: "left" | "right" | "middle", modifiers?: Array<"command" | "control" | "option" | "shift" | "fn">, relative?: boolean }) -> action result
|
||||
|
||||
Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input.
|
||||
|
||||
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Coordinate hovering, clicks, scrolling and dragging require a successful AX observation or screenshot of that window. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.
|
||||
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.
|
||||
|
||||
Hover moves the visible system cursor without pressing a button. It requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.
|
||||
Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.
|
||||
|
||||
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
||||
@@ -346,6 +346,11 @@
|
||||
"y": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"relative": {
|
||||
"description": "Omitted or false uses pixels in the latest screenshot returned by this tool.",
|
||||
"type": "boolean",
|
||||
"const": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -355,6 +360,47 @@
|
||||
"y"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"x": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
},
|
||||
"y": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
},
|
||||
"relative": {
|
||||
"type": "boolean",
|
||||
"const": true,
|
||||
"description": "Use integer thousandths of the window width and height, from 0 to 1000."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"windowId",
|
||||
"x",
|
||||
"y",
|
||||
"relative"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -455,6 +501,11 @@
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"relative": {
|
||||
"description": "Omitted or false uses pixels in the latest screenshot returned by this tool.",
|
||||
"type": "boolean",
|
||||
"const": false
|
||||
},
|
||||
"mouseButton": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
@@ -496,6 +547,80 @@
|
||||
"y"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"x": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
},
|
||||
"y": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
},
|
||||
"relative": {
|
||||
"type": "boolean",
|
||||
"const": true,
|
||||
"description": "Use integer thousandths of the window width and height, from 0 to 1000."
|
||||
},
|
||||
"mouseButton": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"left",
|
||||
"right",
|
||||
"middle"
|
||||
]
|
||||
},
|
||||
"modifiers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"command",
|
||||
"control",
|
||||
"option",
|
||||
"shift",
|
||||
"fn"
|
||||
]
|
||||
}
|
||||
},
|
||||
"clickCount": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "number",
|
||||
"const": 1
|
||||
},
|
||||
{
|
||||
"type": "number",
|
||||
"const": 2
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"windowId",
|
||||
"x",
|
||||
"y",
|
||||
"relative"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -770,49 +895,111 @@
|
||||
"const": "scroll"
|
||||
},
|
||||
"params": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"x": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"y": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"relative": {
|
||||
"description": "Omitted or false uses pixels in the latest screenshot returned by this tool.",
|
||||
"type": "boolean",
|
||||
"const": false
|
||||
},
|
||||
"deltaLinesX": {
|
||||
"type": "integer",
|
||||
"minimum": -10,
|
||||
"maximum": 10
|
||||
},
|
||||
"deltaLinesY": {
|
||||
"type": "integer",
|
||||
"minimum": -10,
|
||||
"maximum": 10
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"windowId",
|
||||
"x",
|
||||
"y",
|
||||
"deltaLinesX",
|
||||
"deltaLinesY"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"x": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"y": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"deltaLinesX": {
|
||||
"type": "integer",
|
||||
"minimum": -10,
|
||||
"maximum": 10
|
||||
},
|
||||
"deltaLinesY": {
|
||||
"type": "integer",
|
||||
"minimum": -10,
|
||||
"maximum": 10
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"x": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
},
|
||||
"y": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
},
|
||||
"relative": {
|
||||
"type": "boolean",
|
||||
"const": true,
|
||||
"description": "Use integer thousandths of the window width and height, from 0 to 1000."
|
||||
},
|
||||
"deltaLinesX": {
|
||||
"type": "integer",
|
||||
"minimum": -10,
|
||||
"maximum": 10
|
||||
},
|
||||
"deltaLinesY": {
|
||||
"type": "integer",
|
||||
"minimum": -10,
|
||||
"maximum": 10
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"windowId",
|
||||
"x",
|
||||
"y",
|
||||
"relative",
|
||||
"deltaLinesX",
|
||||
"deltaLinesY"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"windowId",
|
||||
"x",
|
||||
"y",
|
||||
"deltaLinesX",
|
||||
"deltaLinesY"
|
||||
],
|
||||
"additionalProperties": false
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -829,68 +1016,149 @@
|
||||
"const": "drag"
|
||||
},
|
||||
"params": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"mouseButton": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"left",
|
||||
"right",
|
||||
"middle"
|
||||
]
|
||||
},
|
||||
"modifiers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"command",
|
||||
"control",
|
||||
"option",
|
||||
"shift",
|
||||
"fn"
|
||||
]
|
||||
}
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"relative": {
|
||||
"type": "boolean",
|
||||
"const": false
|
||||
},
|
||||
"fromX": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"fromY": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"toX": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"toY": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"windowId",
|
||||
"fromX",
|
||||
"fromY",
|
||||
"toX",
|
||||
"toY"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"mouseButton": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"left",
|
||||
"right",
|
||||
"middle"
|
||||
]
|
||||
},
|
||||
"modifiers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"command",
|
||||
"control",
|
||||
"option",
|
||||
"shift",
|
||||
"fn"
|
||||
]
|
||||
}
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"fromX": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"fromY": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"toX": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"toY": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"appHandle": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"windowId": {
|
||||
"type": "integer",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 4294967295
|
||||
},
|
||||
"mouseButton": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"left",
|
||||
"right",
|
||||
"middle"
|
||||
]
|
||||
},
|
||||
"modifiers": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"command",
|
||||
"control",
|
||||
"option",
|
||||
"shift",
|
||||
"fn"
|
||||
]
|
||||
}
|
||||
},
|
||||
"snapshotId": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"relative": {
|
||||
"type": "boolean",
|
||||
"const": true
|
||||
},
|
||||
"fromX": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
},
|
||||
"fromY": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
},
|
||||
"toX": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
},
|
||||
"toY": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"windowId",
|
||||
"relative",
|
||||
"fromX",
|
||||
"fromY",
|
||||
"toX",
|
||||
"toY"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"appHandle",
|
||||
"windowId",
|
||||
"fromX",
|
||||
"fromY",
|
||||
"toX",
|
||||
"toY"
|
||||
],
|
||||
"additionalProperties": false
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
@@ -189,7 +189,9 @@ function createApplicationState(socket, record, application, captureBroker, conn
|
||||
data: readFileSync(new URL(customScreenshot.file, import.meta.url)).toString('base64') }
|
||||
: { windowId: params.windowId, mimeType: 'image/png', data: png, pixelWidth: 1, pixelHeight: 1, bounds, coordinateSpace: 'window-points' }
|
||||
if (method === 'click' || method === 'setValue' || method === 'hover') {
|
||||
const observed = [...snapshots.values()].find(snapshot => snapshot.snapshotId === params.snapshotId)
|
||||
const observed = params.snapshotId === undefined && params.elementIndex === undefined
|
||||
? snapshots.get(params.windowId) ?? snapshots.get(null)
|
||||
: [...snapshots.values()].find(snapshot => snapshot.snapshotId === params.snapshotId)
|
||||
if (!observed) refuse('STALE_SNAPSHOT', 'Read fresh state before acting')
|
||||
if (params.windowId !== undefined && observed.windowId !== null && observed.windowId !== params.windowId) refuse('STALE_SNAPSHOT', 'Snapshot belongs to another window')
|
||||
if (observed.axless && params.elementIndex !== undefined) refuse('INVALID_ARGUMENT', 'Element index is absent from the selected snapshot')
|
||||
|
||||
@@ -19,13 +19,13 @@
|
||||
{"type":"tool/ptc-dispatch-start","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:2","name":"cua","arguments":{"method":"getState","params":{"appHandle":"fixture-app","windowId":7}}}}
|
||||
{"type":"tool/ptc-dispatch","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:2","name":"cua","arguments":{"method":"getState","params":{"appHandle":"fixture-app","windowId":7}},"isError":false,"content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"pid\":9,\"windowId\":7,\"snapshotId\":\"fixture-snapshot-1\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 0\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"pid\":9,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}]}}
|
||||
{"type":"tool/ptc-dispatch-start","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:3","name":"cua","arguments":{"method":"screenshot","params":{"appHandle":"fixture-app","windowId":7}}}}
|
||||
{"type":"tool/ptc-dispatch","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:3","name":"cua","arguments":{"method":"screenshot","params":{"appHandle":"fixture-app","windowId":7}},"isError":false,"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"window-points\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}]}}
|
||||
{"type":"tool/ptc-dispatch","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:3","name":"cua","arguments":{"method":"screenshot","params":{"appHandle":"fixture-app","windowId":7}},"isError":false,"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}]}}
|
||||
{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-ptc-native-result"},"toolCallId":"cua-ptc-native-result","content":[{"type":"text","text":"{\n \"appHandle\": \"fixture-app\",\n \"hasIncrement\": true,\n \"pngBytes\": 90,\n \"signature\": \"89504e470d0a1a0a\",\n \"imageWidth\": 1,\n \"imageHeight\": 1\n}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"}
|
||||
{"type":"agent/inbox/spliced","data":{"target":"next-step","start":0,"inserted":[{"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"window-points\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"source":{"kind":"ptc-mode"},"role":"user","id":"{{message:6}}"}]}}
|
||||
{"type":"agent/inbox/spliced","data":{"target":"next-step","start":0,"inserted":[{"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"source":{"kind":"ptc-mode"},"role":"user","id":"{{message:6}}"}]}}
|
||||
{"type":"step/end","data":{"turn":1,"step":1}}
|
||||
{"type":"agent/inbox/spliced","data":{"target":"next-step","start":0,"removedCount":1,"inserted":[]}}
|
||||
{"type":"step/start","data":{"turn":1,"step":2}}
|
||||
{"type":"user/message","data":{"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"window-points\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"source":{"kind":"ptc-mode"},"role":"user","id":"{{message:6}}"},"surfaceOp":"append"}
|
||||
{"type":"user/message","data":{"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"source":{"kind":"ptc-mode"},"role":"user","id":"{{message:6}}"},"surfaceOp":"append"}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"Received native CUA values and one PNG screenshot through PTC. The PNG is 90 bytes with signature 89504e470d0a1a0a."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:7}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791864699463,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"chunk","time":1791864699463,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"Received native CUA values and one PNG screenshot through PTC. The PNG is 90 bytes with signature 89504e470d0a1a0a."}}},{"type":"chunk","time":1791864699463,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791864699463,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"}
|
||||
{"type":"step/end","data":{"turn":1,"step":2}}
|
||||
{"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}}
|
||||
@@ -31,7 +31,7 @@ Use the workflow tool ONLY when the user explicitly asks for a workflow or for l
|
||||
|
||||
Start independent delegations with `subagent` or `subagent_fork` together in one assistant message and continue useful work while they run.
|
||||
|
||||
Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are window-local macOS points, which may differ from screenshot pixels. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
|
||||
Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns AX text and snapshotId. Menu-bar entries are shown by default; includeMenus=true expands their contents. details=true adds identifiers, global-point bounds and complete action names. Both options also apply to getAXState. Each read replaces the indices for that app/window scope. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are pixels in the latest returned screenshot, measured from its top-left. Set relative:true for integer thousandths from 0 to 1000 instead. The Host converts both modes to native window points; AX detail bounds remain global macOS points. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
|
||||
|
||||
Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text separates values and placeholders, and reports subroles, help, URLs, disabled/focused/selected/expanded state and UTF-16 selection ranges when available. Writability is true, false or unknown on relevant controls; omitted state fields do not assert false. selection=unavailable means the reported range exceeds the observed text. Compact output omits ordinary Press/Pick (click), ShowMenu (right click) and ScrollToVisible actions; details=true lists every available action. Use the displayed action name with performSecondaryAction; ambiguous aliases require the complete name from details=true. Truncation reports the reached provider limit. Narrow the window or increase that provider limit before reading again.
|
||||
|
||||
@@ -56,19 +56,19 @@ App and Window methods:
|
||||
- getAXState(options?: { emit?: boolean, disableDiffing?: boolean, includeMenus?: boolean, details?: boolean }) -> string
|
||||
- getScreenshot(options?: { emit?: boolean }) -> Uint8Array
|
||||
- getAXStateAndScreenshot(options?) -> { state: string, screenshot: Uint8Array }
|
||||
- hover(target: number | [x, y]) -> action result
|
||||
- click(target: number | [x, y], options?: { mouseButton?: "left" | "right" | "middle", clickCount?: 1 | 2, modifiers?: Array<"command" | "control" | "option" | "shift" | "fn"> }) -> action result
|
||||
- hover(target: number | [x, y], options?: { relative?: boolean }) -> { dispatched, verified, hoverRoute, systemPointerMoved }
|
||||
- click(target: number | [x, y], options?: { mouseButton?: "left" | "right" | "middle", clickCount?: 1 | 2, modifiers?: Array<"command" | "control" | "option" | "shift" | "fn">, relative?: boolean }) -> action result
|
||||
- setValue(elementIndex: number, value: string) -> action result
|
||||
- selectText(elementIndex: number, text: string, options?: { prefix?: string, suffix?: string, selectionType?: "text" | "cursor_before" | "cursor_after" }) -> action result
|
||||
- performSecondaryAction(elementIndex: number, action: string) -> action result
|
||||
- scroll({ x, y, deltaLinesX, deltaLinesY }) -> action result
|
||||
- drag(from: [x, y], to: [x, y], options?: { mouseButton?: "left" | "right" | "middle", modifiers?: Array<"command" | "control" | "option" | "shift" | "fn"> }) -> action result
|
||||
- scroll({ x, y, deltaLinesX, deltaLinesY, relative? }) -> action result
|
||||
- drag(from: [x, y], to: [x, y], options?: { mouseButton?: "left" | "right" | "middle", modifiers?: Array<"command" | "control" | "option" | "shift" | "fn">, relative?: boolean }) -> action result
|
||||
|
||||
Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a window is hidden or minimized, try app.show() or window.show() and read fresh state. show requests unhide and unminimize without activation, raising or switching Spaces; it chooses the explicit, focused, main or sole AX window. A null windowId means no window was selected for restoration. foregroundChanged compares the front app and top window before and after the request; null means unavailable, and delayed focus changes remain possible. If restoration is unsupported or the window remains unusable, including another Space or Stage Manager, explain the needed foreground step, use app.activate() or an advertised raise action, and read fresh state. Completing the task takes priority when foreground access is necessary. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input.
|
||||
|
||||
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Coordinate hovering, clicks, scrolling and dragging require a successful AX observation or screenshot of that window. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.
|
||||
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. Absolute coordinate input requires a fresh screenshot of that window. Relative input can use a successful AX observation or screenshot. SCREENSHOT_REQUIRED means capture a fresh screenshot before using pixel coordinates. The relative option applies only to coordinate targets, not AX indices. Window resizing since the latest observation rejects coordinate input; movement alone preserves window-local coordinates. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.
|
||||
|
||||
Hover moves the visible system cursor without pressing a button. It requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.
|
||||
Hover first uses advertised AX alternate UI without moving the system cursor, preserving revealed descendants until the pointer target leaves their region. When unavailable, it moves the visible system cursor without pressing a button; that route requires the target point to be unobscured in the foreground app. FOREGROUND_REQUIRED requires explicit activation or raising and a fresh observation before retrying. Returning, AX reads and screenshots do not restore the previous point. Wait for the application's hover delay, observe, then hover into the revealed controls before clicking. Use successive hover calls for intermediate points. The result reports hoverRoute and systemPointerMoved. Other input or application changes can interrupt the interaction; app objects do not reserve applications between calls.
|
||||
|
||||
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. Observation methods emit by default and also return their value. Use void await app.getAXState() to avoid a duplicate REPL completion, or pass emit:false and print the returned text once. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
|
||||
|
||||
|
||||
Reference in new issue
Block a user