Refresh CUA replay for activation and tree guidance
This commit is contained in:
creatixchu committed
1 parent
98698bf5f7
commit
92c73f12e4
4 files changed
+15
-15
No files matched your search
@@ -89,12 +89,12 @@
|
||||
{"type":"step/start","data":{"turn":1,"step":16}}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":16,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:34}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689961,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689961,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947689961,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689961,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
|
||||
{"type":"tool/call","data":{"turn":1,"step":16,"callId":"cua-native-activate-first","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}
|
||||
{"type":"tool/result","data":{"turn":1,"step":16,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-first"},"toolCallId":"cua-native-activate-first","content":[{"type":"text","text":"{\"dispatched\":true,\"foreground\":{\"requestId\":\"cua-native-activate-first\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:35}}"}},"sourceEventSeqs":[89],"surfaceOp":"append"}
|
||||
{"type":"tool/result","data":{"turn":1,"step":16,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-first"},"toolCallId":"cua-native-activate-first","content":[{"type":"text","text":"{\"dispatched\":true,\"foregroundChanged\":true,\"foreground\":{\"requestId\":\"cua-native-activate-first\",\"targetPid\":9,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true}}"}],"isError":false,"id":"{{message:35}}"}},"sourceEventSeqs":[89],"surfaceOp":"append"}
|
||||
{"type":"step/end","data":{"turn":1,"step":16}}
|
||||
{"type":"step/start","data":{"turn":1,"step":17}}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":17,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:36}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947689983,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947689983,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947689983,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947689983,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
|
||||
{"type":"tool/call","data":{"turn":1,"step":17,"callId":"cua-native-activate-repeat","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}
|
||||
{"type":"tool/result","data":{"turn":1,"step":17,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-repeat"},"toolCallId":"cua-native-activate-repeat","content":[{"type":"text","text":"{\"dispatched\":true,\"foreground\":{\"requestId\":\"cua-native-activate-repeat\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":false,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:37}}"}},"sourceEventSeqs":[94],"surfaceOp":"append"}
|
||||
{"type":"tool/result","data":{"turn":1,"step":17,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-native-activate-repeat"},"toolCallId":"cua-native-activate-repeat","content":[{"type":"text","text":"{\"dispatched\":true,\"foregroundChanged\":false,\"foreground\":{\"requestId\":\"cua-native-activate-repeat\",\"targetPid\":9,\"before\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":false}}"}],"isError":false,"id":"{{message:37}}"}},"sourceEventSeqs":[94],"surfaceOp":"append"}
|
||||
{"type":"step/end","data":{"turn":1,"step":17}}
|
||||
{"type":"step/start","data":{"turn":1,"step":18}}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":18,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:38}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690002,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690002,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-native-transition","name":"cua","arguments":"{\"method\":\"pressKey\",\"params\":{\"appHandle\":\"fixture-app\",\"key\":\"Return\"}}"}}},{"type":"chunk","time":1791947690002,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690002,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
|
||||
@@ -244,7 +244,7 @@
|
||||
{"type":"step/start","data":{"turn":1,"step":47}}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":47,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:96}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690714,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690714,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}},{"type":"chunk","time":1791947690714,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690714,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
|
||||
{"type":"tool/call","data":{"turn":1,"step":47,"callId":"cua-paste-activate","name":"cua","arguments":"{\"method\":\"activate\",\"params\":{\"appHandle\":\"fixture-app\"}}"}}
|
||||
{"type":"tool/result","data":{"turn":1,"step":47,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-activate"},"toolCallId":"cua-paste-activate","content":[{"type":"text","text":"{\"dispatched\":true,\"foreground\":{\"requestId\":\"cua-paste-activate\",\"operation\":\"activate\",\"targetPid\":9,\"targetWindowId\":null,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true,\"raiseRequested\":false}}"}],"isError":false,"id":"{{message:97}}"}},"sourceEventSeqs":[244],"surfaceOp":"append"}
|
||||
{"type":"tool/result","data":{"turn":1,"step":47,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-paste-activate"},"toolCallId":"cua-paste-activate","content":[{"type":"text","text":"{\"dispatched\":true,\"foregroundChanged\":true,\"foreground\":{\"requestId\":\"cua-paste-activate\",\"targetPid\":9,\"before\":{\"frontPid\":42,\"focusedWindowId\":7,\"frontWindowId\":99,\"frontWindowPid\":42},\"after\":{\"frontPid\":9,\"focusedWindowId\":7,\"frontWindowId\":7,\"frontWindowPid\":9},\"activationRequested\":true}}"}],"isError":false,"id":"{{message:97}}"}},"sourceEventSeqs":[244],"surfaceOp":"append"}
|
||||
{"type":"step/end","data":{"turn":1,"step":47}}
|
||||
{"type":"step/start","data":{"turn":1,"step":48}}
|
||||
{"type":"assistant/message","data":{"turn":1,"step":48,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:98}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791947690734,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791947690734,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-paste-focus","name":"cua","arguments":"{\"method\":\"getState\",\"params\":{\"appHandle\":\"fixture-app\",\"windowId\":7}}"}}},{"type":"chunk","time":1791947690734,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791947690734,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"}
|
||||
|
||||
@@ -33,7 +33,7 @@ Start independent delegations with `subagent` or `subagent_fork` together in one
|
||||
|
||||
Bash, formatters, code generators, and scripts can overwrite concurrent edits without conflict detection; coordinate their changes explicitly. The agent integrating the work must review the final diff and run relevant tests.
|
||||
|
||||
macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. Use window coordinates for click and mouseMove on macOS. The helper rejects element-targeted pointer requests carrying the window ID supplied by these object methods. For an element’s default action, use invoke with its latest ID. getApp also accepts bundle IDs and .app paths, and findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before activate or raise.
|
||||
macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. Use window coordinates for click and mouseMove on macOS. The helper rejects element-targeted pointer requests carrying the window ID supplied by these object methods. For an element’s default action, use invoke with its latest ID. getApp also accepts bundle IDs and .app paths, and findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.
|
||||
|
||||
Use cua to observe and operate native applications. Always use cua.getCapabilities() to confirm whether a specific API is available on the current platform; its reported capabilities take precedence over availability statements in these instructions. All methods are async; observation options default to { emit: true }, and emit: false suppresses sink output only, leaving returned values unchanged. Actions report dispatch evidence, not verified completion. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI actions and ask the user to unlock the desktop session; null or unknown means verify a usable unlocked desktop session first. cua.getApp requests user consent; if access is denied or revoked, stop using that application until the user explicitly permits another request, and never bypass a refusal or switch tools to avoid it. Never replay uncertain input without a fresh observation.
|
||||
|
||||
@@ -57,7 +57,7 @@ Setup and discovery:
|
||||
Available at app or window level depending on the platform.
|
||||
|
||||
Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, disableDiffing?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- listActions(elementId: string) -> string[] — list the actions an element advertises.
|
||||
- invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none.
|
||||
- performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument.
|
||||
@@ -72,7 +72,7 @@ Use an explicit window for screenshots and pointer input.
|
||||
- scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left.
|
||||
- drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window.
|
||||
|
||||
A string target is an element id; coordinate targets are [x, y] and are pixels in the latest returned screenshot by default. With relative: true they are integer thousandths of the window width and height from 0 to 999, based on the current AX observation or screenshot. Left clicks support one or two presses; right and middle clicks support one. FOREGROUND_REQUIRED means explain the need, then explicitly activate or raise and observe again; activation is never automatic. Read fresh state after actions, and treat WINDOW_CHANGED, STALE_SNAPSHOT, and SCREENSHOT_REQUIRED as requiring the stated fresh observation or capture before any retry.
|
||||
A string target is an element id; coordinate targets are [x, y] and are pixels in the latest returned screenshot by default. With relative: true they are integer thousandths of the window width and height from 0 to 999, based on the current AX observation or screenshot. Left clicks support one or two presses; right and middle clicks support one. FOREGROUND_REQUIRED means explain the need, then call app.activate() and observe again; activation is never automatic. Read fresh state after actions, and treat WINDOW_CHANGED, STALE_SNAPSHOT, and SCREENSHOT_REQUIRED as requiring the stated fresh observation or capture before any retry.
|
||||
|
||||
## Simulated keyboard
|
||||
Available at app or window level depending on the platform.
|
||||
@@ -80,6 +80,6 @@ Available at app or window level depending on the platform.
|
||||
- paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry.
|
||||
- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string.
|
||||
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. Object options such as emit and disableDiffing are facade-only: emit is local output control and neither is a raw RPC parameter, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
|
||||
Maximum click holdMs: 5000 ms.
|
||||
@@ -35,7 +35,7 @@ Start independent delegations with `subagent` or `subagent_fork` together in one
|
||||
|
||||
Bash, formatters, code generators, and scripts can overwrite concurrent edits without conflict detection; coordinate their changes explicitly. The agent integrating the work must review the final diff and run relevant tests.
|
||||
|
||||
macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. Use window coordinates for click and mouseMove on macOS. The helper rejects element-targeted pointer requests carrying the window ID supplied by these object methods. For an element’s default action, use invoke with its latest ID. getApp also accepts bundle IDs and .app paths, and findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before activate or raise.
|
||||
macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. Use window coordinates for click and mouseMove on macOS. The helper rejects element-targeted pointer requests carrying the window ID supplied by these object methods. For an element’s default action, use invoke with its latest ID. getApp also accepts bundle IDs and .app paths, and findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.
|
||||
|
||||
Use cua to observe and operate native applications. Always use cua.getCapabilities() to confirm whether a specific API is available on the current platform; its reported capabilities take precedence over availability statements in these instructions. All methods are async; observation options default to { emit: true }, and emit: false suppresses sink output only, leaving returned values unchanged. Actions report dispatch evidence, not verified completion. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI actions and ask the user to unlock the desktop session; null or unknown means verify a usable unlocked desktop session first. cua.getApp requests user consent; if access is denied or revoked, stop using that application until the user explicitly permits another request, and never bypass a refusal or switch tools to avoid it. Never replay uncertain input without a fresh observation.
|
||||
|
||||
@@ -59,7 +59,7 @@ Setup and discovery:
|
||||
Available at app or window level depending on the platform.
|
||||
|
||||
Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, disableDiffing?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- listActions(elementId: string) -> string[] — list the actions an element advertises.
|
||||
- invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none.
|
||||
- performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument.
|
||||
@@ -74,7 +74,7 @@ Use an explicit window for screenshots and pointer input.
|
||||
- scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left.
|
||||
- drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window.
|
||||
|
||||
A string target is an element id; coordinate targets are [x, y] and are pixels in the latest returned screenshot by default. With relative: true they are integer thousandths of the window width and height from 0 to 999, based on the current AX observation or screenshot. Left clicks support one or two presses; right and middle clicks support one. FOREGROUND_REQUIRED means explain the need, then explicitly activate or raise and observe again; activation is never automatic. Read fresh state after actions, and treat WINDOW_CHANGED, STALE_SNAPSHOT, and SCREENSHOT_REQUIRED as requiring the stated fresh observation or capture before any retry.
|
||||
A string target is an element id; coordinate targets are [x, y] and are pixels in the latest returned screenshot by default. With relative: true they are integer thousandths of the window width and height from 0 to 999, based on the current AX observation or screenshot. Left clicks support one or two presses; right and middle clicks support one. FOREGROUND_REQUIRED means explain the need, then call app.activate() and observe again; activation is never automatic. Read fresh state after actions, and treat WINDOW_CHANGED, STALE_SNAPSHOT, and SCREENSHOT_REQUIRED as requiring the stated fresh observation or capture before any retry.
|
||||
|
||||
## Simulated keyboard
|
||||
Available at app or window level depending on the platform.
|
||||
@@ -82,7 +82,7 @@ Available at app or window level depending on the platform.
|
||||
- paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry.
|
||||
- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string.
|
||||
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. Object options such as emit and disableDiffing are facade-only: emit is local output control and neither is a raw RPC parameter, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
|
||||
Maximum click holdMs: 5000 ms.
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ Start independent delegations with `subagent` or `subagent_fork` together in one
|
||||
|
||||
Bash, formatters, code generators, and scripts can overwrite concurrent edits without conflict detection; coordinate their changes explicitly. The agent integrating the work must review the final diff and run relevant tests.
|
||||
|
||||
macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. Use window coordinates for click and mouseMove on macOS. The helper rejects element-targeted pointer requests carrying the window ID supplied by these object methods. For an element’s default action, use invoke with its latest ID. getApp also accepts bundle IDs and .app paths, and findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before activate or raise.
|
||||
macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. Use window coordinates for click and mouseMove on macOS. The helper rejects element-targeted pointer requests carrying the window ID supplied by these object methods. For an element’s default action, use invoke with its latest ID. getApp also accepts bundle IDs and .app paths, and findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.
|
||||
|
||||
Use cua to observe and operate native applications. Always use cua.getCapabilities() to confirm whether a specific API is available on the current platform; its reported capabilities take precedence over availability statements in these instructions. All methods are async; observation options default to { emit: true }, and emit: false suppresses sink output only, leaving returned values unchanged. Actions report dispatch evidence, not verified completion. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI actions and ask the user to unlock the desktop session; null or unknown means verify a usable unlocked desktop session first. cua.getApp requests user consent; if access is denied or revoked, stop using that application until the user explicitly permits another request, and never bypass a refusal or switch tools to avoid it. Never replay uncertain input without a fresh observation.
|
||||
|
||||
@@ -59,7 +59,7 @@ Setup and discovery:
|
||||
Available at app or window level depending on the platform.
|
||||
|
||||
Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, disableDiffing?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the full tree text; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names.
|
||||
- listActions(elementId: string) -> string[] — list the actions an element advertises.
|
||||
- invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none.
|
||||
- performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument.
|
||||
@@ -74,7 +74,7 @@ Use an explicit window for screenshots and pointer input.
|
||||
- scroll(options: { x: number, y: number, deltaLinesX: number, deltaLinesY: number, relative?: boolean }) -> ActionResult — scroll at a point; deltas are integers from -10 to 10 with at least one nonzero, and positive values reveal content above or left.
|
||||
- drag(from: [number, number], to: [number, number], options?: { mouseButton?: 'left' | 'right' | 'middle', modifiers?: ('command' | 'control' | 'option' | 'shift' | 'fn')[], relative?: boolean }) -> ActionResult — drag between two points of one window.
|
||||
|
||||
A string target is an element id; coordinate targets are [x, y] and are pixels in the latest returned screenshot by default. With relative: true they are integer thousandths of the window width and height from 0 to 999, based on the current AX observation or screenshot. Left clicks support one or two presses; right and middle clicks support one. FOREGROUND_REQUIRED means explain the need, then explicitly activate or raise and observe again; activation is never automatic. Read fresh state after actions, and treat WINDOW_CHANGED, STALE_SNAPSHOT, and SCREENSHOT_REQUIRED as requiring the stated fresh observation or capture before any retry.
|
||||
A string target is an element id; coordinate targets are [x, y] and are pixels in the latest returned screenshot by default. With relative: true they are integer thousandths of the window width and height from 0 to 999, based on the current AX observation or screenshot. Left clicks support one or two presses; right and middle clicks support one. FOREGROUND_REQUIRED means explain the need, then call app.activate() and observe again; activation is never automatic. Read fresh state after actions, and treat WINDOW_CHANGED, STALE_SNAPSHOT, and SCREENSHOT_REQUIRED as requiring the stated fresh observation or capture before any retry.
|
||||
|
||||
## Simulated keyboard
|
||||
Available at app or window level depending on the platform.
|
||||
@@ -82,7 +82,7 @@ Available at app or window level depending on the platform.
|
||||
- paste(text: string, options?: { format?: 'text' | 'html' }) -> ActionResult — deliver text or HTML through the app's Paste command, restoring prior clipboard contents unless another writer changes them; a Paste failure is surfaced without substituting typeText, and PASTE_NOT_CONSUMED means inspect before any retry.
|
||||
- pressKey(key: string, modifiers?: string[]) -> ActionResult — press and release one complete keystroke. For example pressKey('Tab', ['shift']) or pressKey('+'). Key and modifier names are platform-defined strings. Pass key and modifiers separately; do not pack them into a chord string.
|
||||
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. Object options such as emit and disableDiffing are facade-only: emit is local output control and neither is a raw RPC parameter, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name.
|
||||
|
||||
Maximum click holdMs: 5000 ms.
|
||||
|
||||
|
||||
Reference in new issue
Block a user