mward4/deepseek-harness
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 29 Wiki Activity

test(cua): refresh capability snapshots and wait for window creation

Browse Source
This commit is contained in:
creatixchu committed 2026-09-29 19:00:24 +08:00
1 parent 29e70c8ab2
commit b36fe768bb
5 files changed
+202 -19
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/config-catalog.md
+6 -2
View File
@@ -787,7 +787,7 @@ export interface Config {
- `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents`
- `refs`: [`Branded`](../packages/util/brand/src/index.ts) · `Volatile` (`@deepseek-ai/cordis`)
- `source`: [`packages/computer-use/cua/src/index.ts:30`](../packages/computer-use/cua/src/index.ts)
- `source`: [`packages/computer-use/cua/src/index.ts:32`](../packages/computer-use/cua/src/index.ts)
```ts config-catalog
/** Native executable and per-operation resource bounds. */
@@ -807,6 +807,10 @@ export interface Config {
axTimeoutMs: number
/** Maximum bytes retained for one helper response frame. */
maxResponseBytes: number
/** Maximum installed application matches returned for one discovery query. */
maxAppMatches: number
/** Maximum bytes preserved from the system clipboard during one paste. */
maxClipboardBytes: number
/** Maximum UTF-8 bytes admitted for one helper request frame. */
maxRequestBytes: number
/** Duration in milliseconds of one same-window drag. */
@@ -833,7 +837,7 @@ export interface CuaApplicationGrant {
executablePath: string
}
/** Native digest of an executable path and its signing requirement or process lifetime. */
/** Native digest of an executable path and its signing requirement, process lifetime or pending executable identity. */
export type CuaApplicationId = Branded<'CuaApplicationId'>
```
<!-- END GENERATED config-catalog:@deepseek-ai/dsh-cua -->
docs/config-catalog.zh.md
+6 -2
View File
@@ -789,7 +789,7 @@ export interface Config {
- `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents`
- `refs`: [`Branded`](../packages/util/brand/src/index.ts) · `Volatile` (`@deepseek-ai/cordis`)
- `source`: [`packages/computer-use/cua/src/index.ts:30`](../packages/computer-use/cua/src/index.ts)
- `source`: [`packages/computer-use/cua/src/index.ts:32`](../packages/computer-use/cua/src/index.ts)
```ts config-catalog
/** Native executable and per-operation resource bounds. */
@@ -809,6 +809,10 @@ export interface Config {
axTimeoutMs: number
/** Maximum bytes retained for one helper response frame. */
maxResponseBytes: number
/** Maximum installed application matches returned for one discovery query. */
maxAppMatches: number
/** Maximum bytes preserved from the system clipboard during one paste. */
maxClipboardBytes: number
/** Maximum UTF-8 bytes admitted for one helper request frame. */
maxRequestBytes: number
/** Duration in milliseconds of one same-window drag. */
@@ -835,7 +839,7 @@ export interface CuaApplicationGrant {
executablePath: string
}
/** Native digest of an executable path and its signing requirement or process lifetime. */
/** Native digest of an executable path and its signing requirement, process lifetime or pending executable identity. */
export type CuaApplicationId = Branded<'CuaApplicationId'>
```
<!-- END GENERATED config-catalog:@deepseek-ai/dsh-cua -->
packages/computer-use/cua/native/tests/capabilities_e2e.py
+1 -1
View File
@@ -78,7 +78,7 @@ def main():
# The Host consent barrier is separately exercised by provider-lifecycle.spec.ts.
app = call("launchApp", owner)
pid = app["pid"]
wait_for_state(state_file, lambda value: value["pid"] == pid and value["keyWindow"])
wait_for_state(state_file, lambda value: value["pid"] == pid and value["windowId"] > 0)
call("activate", owner)
state = wait_for_state(state_file, lambda value: value["pid"] == pid and value["active"] and value["keyWindow"])
assert call("findApps", {"query": str(application), "maxAppMatches": 20})[0]["pid"] == pid
snapshots/session/cua-native/system-prompt.expected.md
+10 -7
View File
@@ -29,36 +29,39 @@ Use the workflow tool ONLY when the user explicitly asks for a workflow or for l
Start independent subagent delegations together in one assistant message and continue useful work while they run.
Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps and getApp to select an already running application. getApp requests user consent before application observations or input. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. getState returns a complete AX tree and snapshotId. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are window-local macOS points, which may differ from screenshot pixels. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
Use cua with method getCapabilities and params {} to inspect permissions and sessionLocked. If sessionLocked is true or an operation reports SESSION_LOCKED, stop UI attempts and ask the user to unlock macOS. A null sessionLocked means no usable console session was verified. After unlocking, read fresh state; screenshots from a locked desktop are not current observations. Use listApps or cua.getState to discover running applications, and findApps(query) to search installed applications by name, bundle ID or absolute path. getApp requests user consent before launching a stopped application in the background, or before observations and input on a running one. If access is denied or revoked, stop using that application until the user explicitly grants access again. Never switch tools to bypass a refusal. Keep the returned appHandle; every app operation requires it. listWindows returns window IDs and point bounds. Raw getState returns a complete AX tree and snapshotId. Use its element indices only with that snapshotId. Raw screenshot and coordinate operations require a windowId. Coordinates are window-local macOS points, which may differ from screenshot pixels. A verified window can expose no AX elements. Its empty snapshot permits screenshots and coordinate input; element-index actions require an element from a fresh AX observation.
Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object.
Initialize the object API once in persistent PTC. Reuse its variables in later calls; put temporary declarations inside { ... } blocks to avoid redeclaration errors. All object methods are async. getAXState returns text, not a structured state object. Its text marks disabled controls, valueWritable and selectionWritable capabilities, selected state and UTF-16 selection ranges when available.
Use app methods directly for single-window tasks; getApp already reads initial AX state. Call app.getWindows() and await app.getWindow(windowId) only to target a particular window or resolve ambiguous window selection. App screenshots and coordinate actions use the latest AX observation: its focused on-screen, nonzero-size window, otherwise the sole such window. They do not refresh focus automatically. Read app.getAXState() to follow changed focus.
Object API signatures (options default to { emit: true }):
- cua.getCapabilities(options?) -> { platform, permissions, sessionLocked, methods, coordinateSpace, browser }
- cua.listApps(options?) -> Array<{ id, displayName, isRunning, pid }>
- cua.listApps(options?) -> Array<{ id, displayName, isRunning: true, pid }>
- cua.findApps(query: string, options?) -> Array<{ id, displayName, path, isRunning, pid? }>
- cua.getState(options?) -> { apps, browsers: [] }
- cua.getApp(selector: string) -> App
- app.activate() -> { dispatched: true, verified: false }
- app.getWindows(options?) -> Array<{ windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace }>
- app.getWindow(windowId: number) -> Window
- app.typeText(text: string), app.paste(text: string), app.pressKey(chord: string) -> action result
- app.typeText(text: string), app.pressKey(chord: string) -> action result
- app.paste(text: string, options?: { format?: "text" | "html" }) -> action result
App and Window methods:
- getAXState(options?: { emit?: boolean, disableDiffing?: boolean }) -> string
- getScreenshot(options?: { emit?: boolean }) -> Uint8Array
- getAXStateAndScreenshot(options?) -> { state: string, screenshot: Uint8Array }
- click(target: number | [x, y], options?: { mouseButton?: "left" | "right", clickCount?: 1 | 2 }) -> action result
- click(target: number | [x, y], options?: { mouseButton?: "left" | "right" | "middle", clickCount?: 1 | 2, modifiers?: Array<"command" | "control" | "option" | "shift" | "fn"> }) -> action result
- setValue(elementIndex: number, value: string) -> action result
- selectText(elementIndex: number, text: string, options?: { prefix?: string, suffix?: string, selectionType?: "text" | "cursor_before" | "cursor_after" }) -> action result
- performSecondaryAction(elementIndex: number, action: string) -> action result
- scroll({ x, y, deltaLinesX, deltaLinesY }) -> action result
- drag(from: [x, y], to: [x, y]) -> action result
- drag(from: [x, y], to: [x, y], options?: { mouseButton?: "left" | "right" | "middle", modifiers?: Array<"command" | "control" | "option" | "shift" | "fn"> }) -> action result
Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a hidden window, another Space or Stage Manager prevents usable observation, explain the foreground-dependent step before app.activate() or an advertised raise action, then read fresh state. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input.
Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording access in System Settings. After granting access, observe fresh state before continuing; do not automatically replay prior input.
Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
Use selectText only on controls advertising selectionWritable. It selects a unique literal occurrence, or places the cursor before/after it; prefix and suffix must be immediately adjacent. AMBIGUOUS_TEXT requires more context; TEXT_NOT_FOUND or SELECTION_CHANGED requires a fresh observation before deciding on input. Left clicks support single/double clicks; right and middle clicks support single clicks. Modified clicks use pointer input. Keypad names such as KP_0 and KP_Enter retain their physical key identity. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste delivers plain text or HTML through the application's enabled standard Paste menu command. Original clipboard formats are restored if the clipboard still belongs to this paste; a later copy is preserved. Unsupported Paste commands fail without typing fallback. CLIPBOARD_CHANGED means another writer intervened; observe before deciding on further input. Browser applications can be controlled with getApp like other native apps; browser tabs are unavailable. Installed-app search covers standard macOS application directories; use an absolute application path for another location. Launch does not restore a window or document, and cancellation may leave an application already started. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, use the host-provided initialization snippet below once. Its installation-owned file URL works independently of the project directory. Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.
snapshots/session/cua-native/tool-schemas.expected.json
+179 -7
View File
@@ -109,6 +109,34 @@
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"method": {
"type": "string",
"const": "findApps"
},
"params": {
"type": "object",
"properties": {
"query": {
"type": "string",
"minLength": 1,
"pattern": "\\S"
}
},
"required": [
"query"
],
"additionalProperties": false
}
},
"required": [
"method",
"params"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
@@ -207,7 +235,7 @@
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 9007199254740991
"maximum": 4294967295
}
},
"required": [
@@ -251,9 +279,23 @@
"type": "string",
"enum": [
"left",
"right"
"right",
"middle"
]
},
"modifiers": {
"type": "array",
"items": {
"type": "string",
"enum": [
"command",
"control",
"option",
"shift",
"fn"
]
}
},
"clickCount": {
"anyOf": [
{
@@ -284,7 +326,7 @@
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 9007199254740991
"maximum": 4294967295
},
"snapshotId": {
"type": "string",
@@ -302,9 +344,23 @@
"type": "string",
"enum": [
"left",
"right"
"right",
"middle"
]
},
"modifiers": {
"type": "array",
"items": {
"type": "string",
"enum": [
"command",
"control",
"option",
"shift",
"fn"
]
}
},
"clickCount": {
"anyOf": [
{
@@ -377,6 +433,63 @@
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"method": {
"type": "string",
"const": "selectText"
},
"params": {
"type": "object",
"properties": {
"appHandle": {
"type": "string",
"minLength": 1
},
"snapshotId": {
"type": "string",
"minLength": 1
},
"elementIndex": {
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991
},
"text": {
"type": "string",
"minLength": 1
},
"prefix": {
"type": "string"
},
"suffix": {
"type": "string"
},
"selectionType": {
"type": "string",
"enum": [
"text",
"cursor_before",
"cursor_after"
]
}
},
"required": [
"appHandle",
"snapshotId",
"elementIndex",
"text"
],
"additionalProperties": false
}
},
"required": [
"method",
"params"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
@@ -451,6 +564,44 @@
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"method": {
"type": "string",
"const": "paste"
},
"params": {
"type": "object",
"properties": {
"appHandle": {
"type": "string",
"minLength": 1
},
"text": {
"type": "string"
},
"format": {
"type": "string",
"enum": [
"text",
"html"
]
}
},
"required": [
"appHandle",
"text"
],
"additionalProperties": false
}
},
"required": [
"method",
"params"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
@@ -513,7 +664,7 @@
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 9007199254740991
"maximum": 4294967295
},
"snapshotId": {
"type": "string",
@@ -572,7 +723,28 @@
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 9007199254740991
"maximum": 4294967295
},
"mouseButton": {
"type": "string",
"enum": [
"left",
"right",
"middle"
]
},
"modifiers": {
"type": "array",
"items": {
"type": "string",
"enum": [
"command",
"control",
"option",
"shift",
"fn"
]
}
},
"snapshotId": {
"type": "string",
@@ -629,7 +801,7 @@
"windowId": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 9007199254740991
"maximum": 4294967295
}
},
"required": [