From 91f9d14fa8953e1bb3439edeb5aa90514100a567 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 14:00:21 +0800 Subject: [PATCH 01/10] feat(cua): bound accessibility tree reads with typed traversal options --- packages/experimental/cua-macos/README.i18n.yaml | 4 ++-- packages/experimental/cua-macos/README.md | 6 +++--- packages/experimental/cua-macos/README.zh.md | 6 +++--- .../cua-macos/native/protocol.schema.json | 2 +- packages/experimental/cua-macos/src/guidance.ts | 2 +- packages/experimental/cua-macos/src/index.ts | 2 ++ .../cua-macos/tests/loader-composition.spec.ts | 12 ++++++++++-- .../cua-macos/tests/protocol-generation.spec.ts | 1 - .../cua-macos/tests/provider-lifecycle.spec.ts | 2 ++ packages/experimental/cua/README.i18n.yaml | 4 ++-- packages/experimental/cua/README.md | 2 +- packages/experimental/cua/README.zh.md | 2 +- packages/experimental/cua/src/api.ts | 2 ++ packages/experimental/cua/src/guidance.ts | 2 +- packages/experimental/cua/src/index.ts | 6 +++--- packages/experimental/cua/src/native-schema.ts | 2 +- packages/experimental/cua/src/protocol.ts | 11 ++++++++--- packages/experimental/cua/src/types.ts | 8 ++++++-- packages/experimental/cua/tests/api.spec.ts | 16 ++++++++++++++++ packages/experimental/cua/tests/protocol.spec.ts | 8 ++++---- 20 files changed, 69 insertions(+), 31 deletions(-) diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index f8b32baf91..e26f3129a0 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: e51a6c3625873430 - zh: 0a6998897532a05f + en: e25eb1f82970b346 + zh: 418e6d1a9775948d /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: 652b0ce8892e522c zh: 101c20aa18b36c05 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index e20c639e4c..d990054704 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -72,7 +72,7 @@ Source builds default to `sourceSigning: ad-hoc`, without creating certificates | `maxClipboardBytes` | `33554432` | Maximum original clipboard bytes preserved during a paste; oversized contents reject before replacement. | | `pasteReadTimeoutMs` | `1000` | Clipboard read deadline after the Paste command returns; expiry reports `PASTE_NOT_CONSUMED`. | | `maxAppMatches` | `20` | Maximum installed-app search matches; broader queries reject and require a narrower name. | -| `maxElements` / `maxDepth` | `2000` / `25` | Accessibility traversal limits. | +| `maxElements` / `maxDepth` | `5000` / `50` | Accessibility traversal limits. | | `maxClickHoldMs` | `5000` | Maximum explicit click hold in milliseconds; accepts nonnegative integers. Zero only permits immediate release. | | `dragDurationMs` / `dragSteps` | `400` / `20` | Duration and movement segments for a drag. | | `textEventIntervalMs` | `0.25` | Delay after each Unicode down/up event, from 0 to 999 ms. Keystroke timing is independent. | @@ -89,7 +89,7 @@ Application consent is enforced before AX reads, window-title reads, screenshots Raw `getState` exposes validated state fields and the complete text for the requested disclosure options. The native helper also returns structured element diagnostics with opaque string `elementId` values for native checks; the tool projection omits that duplicate array from model text and structured tool values. -AX observations show menu-bar entries by default. Use `app.getAccessibilityTree({ includeMenus: true })` to expand menu contents and `app.getAccessibilityTree({ details: true })` for identifiers, global-point bounds and complete action names. These options also apply to raw `getState`. Every read replaces the application's current snapshot; use element IDs from that read. Details do not bypass the configured node and depth limits. Truncation identifies the reached limit. +AX observations show menu-bar entries by default. Use `app.getAccessibilityTree({ includeMenus: true })` to expand menu contents and `app.getAccessibilityTree({ details: true })` for identifiers, global-point bounds and complete action names. These options also apply to raw `getState`. Every read replaces the application's current snapshot; use element IDs from that read. Pass `maxElements` or `maxDepth` to override either configured limit for one read. Truncation identifies the reached limit. Values and placeholders are separate. Named, stateful or actionable groups and scroll areas retain their own rows. Repeated names, descriptions and help text on one node are shown once. Values, placeholders and URLs keep their distinct meanings even when their text matches; body text shared by distinct nodes remains intact. Writability is explicit on relevant controls, including `AXIncrementor`; `unknown` means the system did not report it. Missing optional state fields do not imply false. UTF-16 ranges outside the observed value are marked unavailable. Compact tree labels can omit AppKit target and selector details. Pass the exact name returned by `listActions(elementId)` to `performAction`. @@ -202,7 +202,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 83177e6047..411b498449 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -72,7 +72,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. | `maxClipboardBytes` | `33554432` | 粘贴时保存原剪贴板内容的字节上限；超限会在替换前拒绝。 | | `pasteReadTimeoutMs` | `1000` | Paste 命令返回后等待剪贴板读取的上限；到期返回 `PASTE_NOT_CONSUMED`。 | | `maxAppMatches` | `20` | 已安装应用搜索的结果上限；范围过大的查询会被拒绝，需要缩小名称范围。 | -| `maxElements` / `maxDepth` | `2000` / `25` | 辅助功能树遍历限制。 | +| `maxElements` / `maxDepth` | `5000` / `50` | 辅助功能树遍历限制。 | | `maxClickHoldMs` | `5000` | 显式点击按住时长的上限，单位为毫秒，接受非负整数。设为 0 时只允许立即松开。 | | `dragDurationMs` / `dragSteps` | `400` / `20` | 拖动时长和移动分段数。 | | `textEventIntervalMs` | `0.25` | 每个 Unicode 按下或抬起事件后的间隔，范围为 0 至 1000 毫秒。击键节奏独立。 | @@ -89,7 +89,7 @@ CUA 工具在读取 AX、窗口标题、截图、激活和输入前检查应用 原始 `getState` 操作返回经过校验的状态字段，以及按本次披露选项生成的完整文本。原生 helper 还返回供原生检查使用的结构化元素诊断数组，其中 `elementId` 是不透明字符串；工具投影会从模型文本和结构化工具值中移除这份重复数组。 -AX 观察默认只展示菜单栏入口。使用 `app.getAccessibilityTree({ includeMenus: true })` 展开菜单内容，使用 `app.getAccessibilityTree({ details: true })` 读取标识符、全局 point 坐标和完整动作名。原始 `getState` 也接受这两个选项。每次读取都会替换应用当前的快照；应使用该次读取返回的元素 ID。详情不会绕过配置的节点数和深度上限；截断提示会指明触发的上限。 +AX 观察默认只展示菜单栏入口。使用 `app.getAccessibilityTree({ includeMenus: true })` 展开菜单内容，使用 `app.getAccessibilityTree({ details: true })` 读取标识符、全局 point 坐标和完整动作名。原始 `getState` 也接受这两个选项。每次读取都会替换应用当前的快照；应使用该次读取返回的元素 ID。传入 `maxElements` 或 `maxDepth` 可调整本次读取的上限。树被截断时会提示触发的上限。 实际值与占位文字分别展示。有名称、状态或可用动作的分组和滚动区域保留独立行。同一节点中重复的名称、描述和帮助只显示一次。实际值、占位文字和 URL 即使文字相同也保留各自含义，不删除不同节点共有的正文。相关控件明确展示可写性，包括 `AXIncrementor`；`unknown` 表示系统未报告。缺少可选状态字段不代表 false。超出当前值长度的 UTF-16 选区标为不可用。树中的动作简称可能省略 AppKit 的 target 和 selector 信息。调用 `performAction` 时使用 `listActions(elementId)` 返回的完整名称。 @@ -202,7 +202,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index 09c7ed9c4e..17b4165d4f 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$comment": "Generated from the shared CUA operation schema. Do not edit.", "title": "dsh-cua native request", - "description": "Host resource fields and private continuations are not model tool parameters. App handles and snapshots belong to one helper; coordinates use window-local native units.", + "description": "The Host resolves resource limits before dispatch. Private continuations are available only to the Host. App handles and snapshots belong to one helper; coordinates use window-local native units.", "oneOf": [ { "type": "object", diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 68243fdaab..4a94798d63 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -5,7 +5,7 @@ import { COMMON_GUIDANCE } from '@deepseek-ai/dsh-cua/guidance' export const PLATFORM_GUIDANCE = 'macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. For pointer input on a sheet, select its window ID with app.getWindow(id). For menu items outside a window, use app.invoke(id). findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.' const methodDetails = new Map([ - ['getAccessibilityTree', 'If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'], + ['getAccessibilityTree', 'macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'], ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again.'], ['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'], ]) diff --git a/packages/experimental/cua-macos/src/index.ts b/packages/experimental/cua-macos/src/index.ts index fb3fada07e..77eb2e4a0e 100644 --- a/packages/experimental/cua-macos/src/index.ts +++ b/packages/experimental/cua-macos/src/index.ts @@ -43,6 +43,8 @@ export interface Config extends CuaConfig { /** Validate shared operation bounds and macOS source signing before preparing the helper. */ export const Config = Schema.object({ ...CuaConfigFields, + maxElements: CuaConfigFields.maxElements.default(5000), + maxDepth: CuaConfigFields.maxDepth.default(50), permissionPollIntervalMs: Schema.number().step(1).min(100).max(2 ** 31 - 1).default(1000), sourceSigning: Schema.union([ Schema.const('ad-hoc'), diff --git a/packages/experimental/cua-macos/tests/loader-composition.spec.ts b/packages/experimental/cua-macos/tests/loader-composition.spec.ts index e92f138d87..45a5b096a5 100644 --- a/packages/experimental/cua-macos/tests/loader-composition.spec.ts +++ b/packages/experimental/cua-macos/tests/loader-composition.spec.ts @@ -134,7 +134,7 @@ it.skipIf(process.platform === 'win32')('shares one application between retained const toolCalls = agent.session.snapshotEvents().filter(event => event.type === 'tool/call') expect(events.filter(event => event.event === 'call').map(event => event.requestId)).toEqual(toolCalls.map(event => event.data.callId)) expect(events.find(event => event.event === 'state')?.count).toBe(1) - expect(events.find(event => event.method === 'getState')?.params).toMatchObject({ maxElements: 2000, maxDepth: 25 }) + expect(events.find(event => event.method === 'getState')?.params).toMatchObject({ maxElements: 5000, maxDepth: 50 }) expect(events.find(event => event.method === 'click')?.params).toMatchObject({ holdMs: 800 }) expect(toolCalls[3]?.data.arguments).toBe(JSON.stringify(calls[3])) @@ -173,7 +173,15 @@ it.skipIf(process.platform === 'win32')('shares one application between retained }) const retainedApp = await objectApi(agent).getApp('test.fixture') const retainedWindow = await retainedApp.getWindow(7) - expect(await retainedWindow.getAccessibilityTree({ emit: false })).toContain('Count: 1') + await retainedApp.getAccessibilityTree({ maxDepth: 60 }) + expect(await retainedWindow.getAccessibilityTree({ maxElements: 4000, emit: false })).toContain('Count: 1') + await retainedWindow.getAccessibilityTree({ emit: false }) + const reads = (await readEvents()).filter(event => event.method === 'getState').slice(-3) + expect(reads.map(event => event.params)).toEqual([ + { appHandle: retainedApp.appHandle, maxElements: 5000, maxDepth: 60 }, + { appHandle: retainedApp.appHandle, windowId: 7, maxElements: 4000, maxDepth: 50 }, + { appHandle: retainedApp.appHandle, windowId: 7, maxElements: 5000, maxDepth: 50 }, + ]) const peer = await context.agents.create({ sessionId: SessionId('native-loader-peer'), agentOptions: { provider: 'native-fixture', model: 'vision' } }) const peerApp = await objectApi(peer.agent).getApp('test.fixture') const peerWindow = await peerApp.getWindow(7) diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index 36c6dd94c9..b3830cbc9a 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -29,7 +29,6 @@ describe('native protocol generation', () => { }) it.each([ - { method: 'getState', params: { appHandle: 'app', maxElements: 1 } }, { method: 'drag', params: { appHandle: 'app', windowId: 1, fromX: 1, fromY: 1, toX: 2, toY: 2, steps: 1 } }, { method: 'paste', params: { appHandle: 'app', text: 'text', maxClipboardBytes: 1024 } }, { method: 'paste', params: { appHandle: 'app', text: 'text', pasteReadTimeoutMs: 1000 } }, diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index 5e7b5893f6..c610af3f03 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -94,6 +94,8 @@ it('projects shared controls and macOS signing without adding signing to the sha for (const field of ['sourceSigning', 'permissionPollIntervalMs', 'agentPreset', 'helperPath']) { expect(projected.schema.anyOf?.[0]).toHaveProperty(`properties.${field}`) } + expect(CuaConfig({})).toMatchObject({ maxElements: 2000, maxDepth: 25 }) + expect(NativeProvider.Config({})).toMatchObject({ maxElements: 5000, maxDepth: 50 }) expect(CuaConfig({})).not.toHaveProperty('sourceSigning') expect(CuaConfig.dict).not.toHaveProperty('sourceSigning') expect(NativeProvider.Config({ sourceSigning: 'ad-hoc', timeoutMs: 1234 })).toMatchObject({ diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index 40d8ac79c6..1a21e815da 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -15,8 +15,8 @@ en: 5db4d4fea25493b9 zh: 65ab64fc24eb0d09 /deepseek-ai-dsh-cua/use-this-package: - en: ba1f8e6ae51f69d7 - zh: 8f4fde9857bae319 + en: 1e00d35a1843e0ea + zh: 5e4e6a6567f8687e /deepseek-ai-dsh-cua/use-this-package/permission-controls: en: d92d5948a7941e8d zh: d43491ae6521b87e diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 1ee0a555d1..10a257f05a 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -31,7 +31,7 @@ Application consent supplies English, Chinese, and Japanese display text from `s Mount the [macOS provider](../cua-macos/README.md) to enable native operations on macOS. External PTC hosts import `connectCua` from `@deepseek-ai/dsh-cua/bridge` and supply the current `tools.cua` binding. Source compositions and native preparation belong to the provider. -Use `cua.getApp(selector)` to select an application; it obtains consent, optionally launches a stopped application in the background, and reads its initial accessibility tree silently when the provider supports application-level accessibility. `cua.listApps` and `cua.getState` list running applications; `cua.findApps(query)` searches installed applications, including stopped ones. `app.activate()`, `app.getWindows()`, and `app.getWindow(windowId)` operate on the selected application; `getWindow` returns the object for screenshots and pointer input. Select an application and one of its windows before any capture or pointer input. Call `app.getAccessibilityTree()` to emit the tree. Within one API instance, the facade keeps one current accessibility snapshot per application handle across app and window objects, including repeated selections; each new read invalidates the previous snapshot. Element operations use the latest snapshot ID, so callers use IDs from the latest tree and do not assume an older ID still identifies the same control. Element IDs are opaque strings. +Use `cua.getApp(selector)` to select an application; it obtains consent, optionally launches a stopped application in the background, and reads its initial accessibility tree silently when the provider supports application-level accessibility. `cua.listApps` and `cua.getState` list running applications; `cua.findApps(query)` searches installed applications, including stopped ones. `app.activate()`, `app.getWindows()`, and `app.getWindow(windowId)` operate on the selected application; `getWindow` returns the object for screenshots and pointer input. Select an application and one of its windows before any capture or pointer input. Call `app.getAccessibilityTree()` to emit the tree. App and window reads accept `maxElements` and `maxDepth` independently. Omitted limits use the provider configuration. Within one API instance, the facade keeps one current accessibility snapshot per application handle across app and window objects, including repeated selections; each new read invalidates the previous snapshot. Element operations use the latest snapshot ID, so callers use IDs from the latest tree and do not assume an older ID still identifies the same control. Element IDs are opaque strings. Keyboard methods are available in the scopes a provider declares; the provider owns how input is delivered within each scope. A lost connection invalidates its handles and coordinates. Failed cleanup is reported before that connection is retired; select the application again and inspect fresh state, because earlier input may have partially executed. Screenshot image errors retain the failing stage and attachment error code. Invalid request arguments reject before dispatch with a readable method, parameter and requirement. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index b8592b36eb..cd8590b1c4 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -31,7 +31,7 @@ kind: "package-reference" 挂载 [macOS 提供方](../cua-macos/README.zh.md)以在 macOS 上启用原生操作。外部 PTC 宿主从 `@deepseek-ai/dsh-cua/bridge` 导入 `connectCua`，并提供当前 `tools.cua` 绑定。源码组合与原生组件准备由提供方负责。 -使用 `cua.getApp(selector)` 选择应用；它会获取授权、按需在后台启动已停止的应用，并在 provider 支持应用级辅助功能时静默读取初始辅助功能树。`cua.listApps` 和 `cua.getState` 列出运行中的应用；`cua.findApps(query)` 搜索已安装应用，包括未运行的应用。`app.activate()`、`app.getWindows()` 和 `app.getWindow(windowId)` 操作选定的应用；`getWindow` 返回用于截图和指针输入的对象。进行任何截图或指针输入前，必须选择应用及其一个窗口。调用 `app.getAccessibilityTree()` 输出树。在单个 API 实例内，门面按应用句柄保存一份当前辅助功能快照，该句柄的所有应用对象和窗口对象共享它，包括重复选择；每次新读取都会使上一份快照失效。元素操作使用最新快照 ID，因此调用方应使用最新树中的元素 ID，不应假定旧 ID 仍指向同一控件。元素 ID 是不透明字符串。 +使用 `cua.getApp(selector)` 选择应用；它会获取授权、按需在后台启动已停止的应用，并在 provider 支持应用级辅助功能时静默读取初始辅助功能树。`cua.listApps` 和 `cua.getState` 列出运行中的应用；`cua.findApps(query)` 搜索已安装应用，包括未运行的应用。`app.activate()`、`app.getWindows()` 和 `app.getWindow(windowId)` 操作选定的应用；`getWindow` 返回用于截图和指针输入的对象。进行任何截图或指针输入前，必须选择应用及其一个窗口。调用 `app.getAccessibilityTree()` 输出树。读取应用或窗口时，可以单独传入 `maxElements` 或 `maxDepth`。省略的参数使用当前平台配置。在单个 API 实例内，门面按应用句柄保存一份当前辅助功能快照，该句柄的所有应用对象和窗口对象共享它，包括重复选择；每次新读取都会使上一份快照失效。元素操作使用最新快照 ID，因此调用方应使用最新树中的元素 ID，不应假定旧 ID 仍指向同一控件。元素 ID 是不透明字符串。 键盘方法可在 provider 声明的作用域中使用；每个作用域内的输入投递由 provider 负责。连接丢失会使其句柄和坐标失效。清理失败会先报告，再释放该连接；应重新选择应用并读取新状态，因为先前的输入可能已部分执行。截图图像错误会保留失败阶段和附件错误码。请求参数校验失败时，在分派前返回包含方法、参数和要求的可读消息。 diff --git a/packages/experimental/cua/src/api.ts b/packages/experimental/cua/src/api.ts index 5dee39030f..bc12b1163c 100644 --- a/packages/experimental/cua/src/api.ts +++ b/packages/experimental/cua/src/api.ts @@ -126,6 +126,8 @@ function createCuaObjects(call: ReturnType, sink: CuaObser requireAccessibility() observed.delete(target.appHandle) const state = StateResult.parse(await scopedCall('getState', { + ...options?.maxElements === undefined ? {} : { maxElements: options.maxElements }, + ...options?.maxDepth === undefined ? {} : { maxDepth: options.maxDepth }, ...options?.includeMenus === undefined ? {} : { includeMenus: options.includeMenus }, ...options?.details === undefined ? {} : { details: options.details }, })) diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index 49caa08bf2..22f623649f 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -24,7 +24,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index 43ca13308e..e4f3ec8313 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -77,9 +77,9 @@ export interface CuaConfig { textEventIntervalMs: number /** Native connection cleanup, socket-write and launcher termination budget in milliseconds. */ graceMs: number - /** Maximum nodes admitted by a native accessibility snapshot. */ + /** Default node limit for accessibility reads and Paste menu discovery. */ maxElements: number - /** Maximum depth traversed by a native accessibility snapshot. */ + /** Default depth limit for accessibility reads and Paste menu discovery. */ maxDepth: number } @@ -230,7 +230,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) ? { ...params, maxClipboardBytes: config.maxClipboardBytes, pasteReadTimeoutMs: config.pasteReadTimeoutMs, maxElements: config.maxElements, maxDepth: config.maxDepth } : method === 'getState' - ? { ...params, maxElements: config.maxElements, maxDepth: config.maxDepth } + ? { maxElements: config.maxElements, maxDepth: config.maxDepth, ...params } : method === 'drag' ? { ...params, durationMs: config.dragDurationMs, steps: config.dragSteps } : params const requestNative = () => client.request(method, nativeParams, signal, execution.callId) diff --git a/packages/experimental/cua/src/native-schema.ts b/packages/experimental/cua/src/native-schema.ts index 5bb061383f..fae68ead2c 100644 --- a/packages/experimental/cua/src/native-schema.ts +++ b/packages/experimental/cua/src/native-schema.ts @@ -78,7 +78,7 @@ export function nativeProtocol( $schema: 'https://json-schema.org/draft/2020-12/schema', $comment: 'Generated from the shared CUA operation schema. Do not edit.', title: 'dsh-cua native request', - description: 'Host resource fields and private continuations are not model tool parameters. App handles and snapshots belong to one helper; coordinates use window-local native units.', + description: 'The Host resolves resource limits before dispatch. Private continuations are available only to the Host. App handles and snapshots belong to one helper; coordinates use window-local native units.', oneOf: schemas, }, null, 2) + '\n' } } diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index 3bedc404d1..c78008f14e 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -7,6 +7,10 @@ const id = z.number().int().positive().max(2 ** 32 - 1) const pid = z.number().int().positive().max(2 ** 31 - 1) const point = z.number().nonnegative() const relativePoint = z.number().int().min(0).max(999) +const traversalLimits = z.object({ + maxElements: z.number().int().min(1).max(10000), + maxDepth: z.number().int().min(1).max(100), +}).strict() const app = { appHandle: z.string().min(1) } const pointer = { mouseButton: z.enum(['left', 'right', 'middle']).optional(), modifiers: z.array(z.enum(['command', 'control', 'option', 'shift', 'fn'])).optional() } @@ -50,6 +54,7 @@ function requestSchema(hover: T) { operation('getState', { ...app, windowId: id.optional(), includeMenus: z.boolean().optional().describe('Expand menu contents; omitted or false shows menu-bar entries only.'), details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'), + ...traversalLimits.partial().shape, }), operation('listActions', snapshot), operation('performAction', { ...snapshot, action: z.string().min(1), argument: jsonValue.optional() }), @@ -91,12 +96,12 @@ export const SystemPointerRequest = requestSchema(z.union(MouseMoveTargets.map(t useSystemPointer: z.boolean().optional().describe('Request use of the system pointer. The Host checks permission before dispatch.'), })))) -/** Resource parameters supplied by the Host, never accepted from model tool arguments. */ +/** Native resource fields. The Host fills omitted observation limits from the provider configuration. */ export const NativeResourceParameters = { - getState: z.object({ maxElements: z.number().int().min(1).max(10000), maxDepth: z.number().int().min(1).max(100) }).strict(), + getState: traversalLimits, drag: z.object({ durationMs: z.number().int().min(1).max(5000), steps: z.number().int().min(1).max(240) }).strict(), paste: z.object({ maxClipboardBytes: z.number().int().min(1024), pasteReadTimeoutMs: z.number().int().min(1).max(2 ** 31 - 1), - maxElements: z.number().int().min(1).max(10000), maxDepth: z.number().int().min(1).max(100), + ...traversalLimits.shape, }).strict(), findApps: z.object({ maxAppMatches: z.number().int().min(1).max(1000) }).strict(), } diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 98c7c7a1a2..8f37862aaf 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -38,6 +38,10 @@ export interface CuaObservationOptions { emit?: boolean } export interface CuaAxOptions extends CuaObservationOptions { /** Reserved compatibility option; observations are never diffs. */ disableDiffing?: boolean + /** Maximum nodes visited for this read, from 1 to 10000. Omission uses the provider setting. */ + maxElements?: number + /** Maximum traversal depth for this read, from 1 to 100. Omission uses the provider setting. */ + maxDepth?: number /** Expand menu contents. The default shows menu-bar entries only. */ includeMenus?: boolean /** Include identifiers, global native bounds and complete action names. */ @@ -103,8 +107,8 @@ export interface CuaSurface { /** * Read the full accessibility tree, with menu contents collapsed by default. * The returned tree becomes the application's current snapshot; use element ids from its latest successful read. - * @param options - emission, menu expansion and field detail options. - * @returns Full accessibility tree text with element ids, subject to the provider's traversal limits. + * @param options - Output, traversal limits, menu expansion and field detail options. + * @returns Accessibility tree text with element ids, up to the limits for this read. */ getAccessibilityTree(options?: CuaAxOptions): Promise /** diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index f71b958368..437a6cb7df 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -912,3 +912,19 @@ it('forwards provider options while retaining the selected target and coordinate appHandle: 'app-17', snapshotId: 'snapshot-2', windowId: 23, x: 250, y: 750, relative: true, trackingMode: 'coarse', } }) }) + +it('forwards bounded traversal limits on a window-only provider', async () => { + const f = fixture() + f.setCapabilities({ ...capabilities, platform: 'windows', appLevelAccessibility: false }) + const app = await f.api.getApp('Editor') + const win = await app.getWindow(23) + const before = f.dispatch.mock.calls.length + for (const limits of [{ maxDepth: 0 }, { maxDepth: 101 }, { maxElements: 1.5 }, { maxElements: 10001 }]) { + await expect(win.getAccessibilityTree(limits)).rejects.toThrow(/maxDepth|maxElements/) + } + expect(f.dispatch).toHaveBeenCalledTimes(before) + await win.getAccessibilityTree({ maxDepth: 1, maxElements: 1 }) + expect(f.dispatch).toHaveBeenLastCalledWith({ + method: 'getState', params: { appHandle: app.appHandle, windowId: 23, maxDepth: 1, maxElements: 1 }, + }) +}) diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index f259df5fcd..338d8e6752 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -6,7 +6,7 @@ import { CapabilitiesResult, Request, ScreenshotResult, SystemPointerRequest } f const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [ { reason: 'raw process ID instead of a selected app handle', request: { method: 'typeText', params: { pid: 17, text: 'hello' } } }, { reason: 'unknown native method', request: { method: 'shutdown', params: {} } }, - { reason: 'host-owned traversal limit', request: { method: 'getState', params: { appHandle: 'app', maxElements: 1000000 } } }, + { reason: 'node limit above the accepted range', request: { method: 'getState', params: { appHandle: 'app', maxElements: 1000000 } } }, { reason: 'extra tool argument', request: { method: 'listApps', params: {}, helperPath: '/untrusted/helper' } }, { reason: 'element click without an observed snapshot', request: { method: 'click', params: { appHandle: 'app', elementId: '0' } } }, { reason: 'numeric element id', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: 0 } } }, @@ -66,11 +66,11 @@ it.each([ }).success).toBe(false) }) -it('accepts disclosure options without accepting host traversal limits', () => { - const params = { appHandle: 'app', includeMenus: true, details: false } +it('accepts disclosure options and bounded per-read traversal limits', () => { + const params = { appHandle: 'app', includeMenus: true, details: false, maxDepth: 100, maxElements: 10000 } expect(Request.parse({ method: 'getState', params })).toEqual({ method: 'getState', params }) expect(Request.safeParse({ method: 'getState', params: { ...params, details: 'all' } }).success).toBe(false) - expect(Request.safeParse({ method: 'getState', params: { ...params, maxDepth: 100 } }).success).toBe(false) + expect(Request.safeParse({ method: 'getState', params: { ...params, maxDepth: 101 } }).success).toBe(false) }) it.each([ From 47254be6f76285325507284209e74ef71d4a14a4 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 14:02:28 +0800 Subject: [PATCH 02/10] test(cua): capture typed traversal limits in model guidance and tool schemas --- snapshots/session/cua-native/system-prompt.expected.md | 2 +- .../session/cua-native/tool-schemas.expected.json | 10 ++++++++++ snapshots/session/cua-ptc/system-prompt.expected.md | 2 +- .../ptc-script-retained/system-prompt.expected.md | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 9589c20d8e..74b65450b8 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -57,7 +57,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index b9bb435797..ed09a6bb40 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -304,6 +304,16 @@ "details": { "description": "Include identifiers, global native bounds and complete action names.", "type": "boolean" + }, + "maxElements": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "maxDepth": { + "type": "integer", + "minimum": 1, + "maximum": 100 } }, "required": [ diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index 6888ad0481..e880caf155 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -59,7 +59,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index c670f443b9..fadc73733b 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -59,7 +59,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean }) -> string — read the tree within the provider limits; includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. If the tree reports maxElements or maxDepth truncation, select a narrower window or ask the user to raise the named macOS provider setting and read again. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. From 6b3ed06115c498fb9ca451dfa5f328414f4974d9 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 14:12:43 +0800 Subject: [PATCH 03/10] feat(cua-macos): raise AX traversal ceilings and document them per platform --- packages/experimental/cua-macos/README.md | 2 +- packages/experimental/cua-macos/README.zh.md | 2 +- .../native/Sources/CuaNative/AXAccess.swift | 21 +++++++++++-------- .../native/Sources/CuaNative/AXWorker.swift | 4 ++-- .../CuaNative/NativeProtocol.generated.swift | 2 ++ .../cua-macos/native/generate-protocol.ts | 10 +++++++++ .../cua-macos/native/protocol.schema.json | 8 +++---- .../experimental/cua-macos/src/ax-limits.ts | 2 ++ .../experimental/cua-macos/src/guidance.ts | 2 +- packages/experimental/cua-macos/src/index.ts | 5 +++-- .../tests/loader-composition.spec.ts | 8 +++---- .../tests/protocol-generation.spec.ts | 10 +++++++++ .../tests/provider-lifecycle.spec.ts | 4 ++++ packages/experimental/cua/src/guidance.ts | 2 +- packages/experimental/cua/src/protocol.ts | 9 ++++---- packages/experimental/cua/src/types.ts | 4 ++-- packages/experimental/cua/tests/api.spec.ts | 2 +- .../experimental/cua/tests/protocol.spec.ts | 6 +++--- 18 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 packages/experimental/cua-macos/src/ax-limits.ts diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index d990054704..aeb34b100b 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -202,7 +202,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 411b498449..c6ed84e59c 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -202,7 +202,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift index 3b39ab1f02..46a9c1e3b4 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXAccess.swift @@ -570,12 +570,16 @@ final class AXAccess { var lines = ["AX scope=\(requested == nil ? "application" : "window") offsets=UTF-16\(details ? " bounds=global-points" : "")"] var elements: [JSONValue] = [] var indexed: [Int: AXUIElement] = [:] - func walk(_ element: AXUIElement, level: Int, depth: Int) throws { + var pending = roots.reversed().map { (element: $0, level: 0, depth: 0) } + func enqueueChildren(_ children: [AXUIElement], level: Int, depth: Int) { + for child in children.reversed() { pending.append((child, level, depth)) } + } + while let (element, level, depth) = pending.popLast() { try cancellation.check() let hash = CFHash(element) - if visited[hash, default: []].contains(where: { CFEqual($0, element) }) { return } - guard count < maxElements else { elementLimitReached = true; return } - guard depth <= maxDepth else { depthLimitReached = true; return } + if visited[hash, default: []].contains(where: { CFEqual($0, element) }) { continue } + guard count < maxElements else { elementLimitReached = true; break } + guard depth <= maxDepth else { depthLimitReached = true; continue } visited[hash, default: []].append(element) count += 1 let observation = try nodeObservation(attributes: metadata(element), fallback: { try self.attribute(element, $0) }) @@ -609,8 +613,8 @@ final class AXAccess { identifier: identifier, url: url, focused: focused, expanded: expanded) node.validateSelection() if node.transparent || (!addressable && !node.hasMeaning && role != "AXWindow" && role != "AXSheet") { - for child in try childValues(observed("AXChildren"), attribute: "AXChildren") { try walk(child, level: level, depth: depth + 1) } - return + enqueueChildren(try childValues(observed("AXChildren"), attribute: "AXChildren"), level: level, depth: depth + 1) + continue } let index: Int? = addressable ? indexed.count : nil if let index { indexed[index] = element } @@ -633,10 +637,9 @@ final class AXAccess { "selectedRange": node.selectedRange.map { .array([.number(Double($0.location)), .number(Double($0.length))]) } ?? .null, ])) } - if role == "AXMenuBarItem" && !includeMenus { menusCollapsed = true; return } - for child in try childValues(observed("AXChildren"), attribute: "AXChildren") { try walk(child, level: level + 1, depth: depth + 1) } + if role == "AXMenuBarItem" && !includeMenus { menusCollapsed = true; continue } + enqueueChildren(try childValues(observed("AXChildren"), attribute: "AXChildren"), level: level + 1, depth: depth + 1) } - for root in roots { try walk(root, level: 0, depth: 0) } if menusCollapsed { lines.append("Menu contents omitted; request includeMenus=true to expand.") } let truncated = elementLimitReached || depthLimitReached if truncated { lines.append(AXTextFormatter.truncation(maxElements: elementLimitReached ? maxElements : nil, maxDepth: depthLimitReached ? maxDepth : nil)) } diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift index 5750a18b6d..ed95ff236d 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/AXWorker.swift @@ -304,7 +304,7 @@ actor AXWorker { let projection: AXProjection do { projection = try ax().project(identity: identity, windowID: window, - maxElements: params.integer("maxElements", in: 1...10000), maxDepth: params.integer("maxDepth", in: 1...100), + maxElements: params.integer("maxElements", in: NativeProtocol.elementRange), maxDepth: params.integer("maxDepth", in: NativeProtocol.depthRange), includeMenus: params.optionalBoolean("includeMenus") ?? false, details: params.optionalBoolean("details") ?? false) } catch let failure as NativeFailure { throw Self.observationFailure(failure) @@ -340,7 +340,7 @@ actor AXWorker { return .paste(PastePlan(handle: handle, text: try params.string("text"), format: try params.optionalString("format") ?? "text", maxBytes: try params.integer("maxClipboardBytes", in: 1024...Int.max), readTimeout: .milliseconds(try params.integer("pasteReadTimeoutMs", in: 1...2147483647)), - maxElements: try params.integer("maxElements", in: 1...10000), maxDepth: try params.integer("maxDepth", in: 1...100))) + maxElements: try params.integer("maxElements", in: NativeProtocol.elementRange), maxDepth: try params.integer("maxDepth", in: NativeProtocol.depthRange))) case "sendKeyEvent": try inputs().0.keyDown(params.string("key"), app: handle, run: params.string("runId")) { try ax().keyboardRecipient(identity, catalog: catalog) diff --git a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift index 564a9b669f..0f038424cb 100644 --- a/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift +++ b/packages/experimental/cua-macos/native/Sources/CuaNative/NativeProtocol.generated.swift @@ -1,5 +1,7 @@ // Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit. enum NativeProtocol { + static let elementRange = 1...100000 + static let depthRange = 1...512 static let fields: [String: Set] = [ "activate": ["appHandle"], "applicationIcons": ["paths"], diff --git a/packages/experimental/cua-macos/native/generate-protocol.ts b/packages/experimental/cua-macos/native/generate-protocol.ts index a97828256b..6e1e12186c 100644 --- a/packages/experimental/cua-macos/native/generate-protocol.ts +++ b/packages/experimental/cua-macos/native/generate-protocol.ts @@ -1,6 +1,8 @@ /** Generate macOS admission artifacts from the shared CUA operation definitions. */ import { readFile, writeFile } from 'node:fs/promises' import { z } from 'zod' +import { AX_TRAVERSAL_MAX } from '../src/ax-limits.ts' +import { NativeResourceParameters } from '@deepseek-ai/dsh-cua/protocol' import { nativeProtocol } from '@deepseek-ai/dsh-cua/native-schema' import { PermissionGuideParameters, PermissionSettingsParameters } from '../src/permission-guide.ts' @@ -9,7 +11,13 @@ import { PermissionGuideParameters, PermissionSettingsParameters } from '../src/ * @returns Native JSON Schema and Swift field/enum tables, including private continuations. */ export function renderNativeProtocol(): { json: string; swift: string } { + const traversal = { + maxElements: z.number().int().min(1).max(AX_TRAVERSAL_MAX.maxElements), + maxDepth: z.number().int().min(1).max(AX_TRAVERSAL_MAX.maxDepth), + } const { fields, enums, json } = nativeProtocol({ + getState: NativeResourceParameters.getState.extend(traversal), + paste: NativeResourceParameters.paste.extend(traversal), sendKeyEvent: z.object({ runId: z.string().min(1) }), mouseMove: z.object({ useSystemPointer: z.boolean().optional() }), requestPermission: PermissionGuideParameters, openPermissionSettings: PermissionSettingsParameters, @@ -20,6 +28,8 @@ export function renderNativeProtocol(): { json: string; swift: string } { const swift = [ '// Generated by native/generate-protocol.ts from @deepseek-ai/dsh-cua/native-schema. Do not edit.', 'enum NativeProtocol {', + ` static let elementRange = 1...${AX_TRAVERSAL_MAX.maxElements}`, + ` static let depthRange = 1...${AX_TRAVERSAL_MAX.maxDepth}`, ' static let fields: [String: Set] = [', ...[...fields].sort(([a], [b]) => a.localeCompare(b)).map(([method, names]) => ` ${JSON.stringify(method)}: [${quoted(names)}],`), ' ]', diff --git a/packages/experimental/cua-macos/native/protocol.schema.json b/packages/experimental/cua-macos/native/protocol.schema.json index 17b4165d4f..84f62bbef4 100644 --- a/packages/experimental/cua-macos/native/protocol.schema.json +++ b/packages/experimental/cua-macos/native/protocol.schema.json @@ -250,12 +250,12 @@ "maxElements": { "type": "integer", "minimum": 1, - "maximum": 10000 + "maximum": 100000 }, "maxDepth": { "type": "integer", "minimum": 1, - "maximum": 100 + "maximum": 512 } }, "required": [ @@ -944,12 +944,12 @@ "maxElements": { "type": "integer", "minimum": 1, - "maximum": 10000 + "maximum": 100000 }, "maxDepth": { "type": "integer", "minimum": 1, - "maximum": 100 + "maximum": 512 } }, "required": [ diff --git a/packages/experimental/cua-macos/src/ax-limits.ts b/packages/experimental/cua-macos/src/ax-limits.ts new file mode 100644 index 0000000000..b45ba0e599 --- /dev/null +++ b/packages/experimental/cua-macos/src/ax-limits.ts @@ -0,0 +1,2 @@ +/** Largest traversal budgets accepted by the macOS helper. */ +export const AX_TRAVERSAL_MAX = { maxElements: 100_000, maxDepth: 512 } as const diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 4a94798d63..1ed115743f 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -5,7 +5,7 @@ import { COMMON_GUIDANCE } from '@deepseek-ai/dsh-cua/guidance' export const PLATFORM_GUIDANCE = 'macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. For pointer input on a sheet, select its window ID with app.getWindow(id). For menu items outside a window, use app.invoke(id). findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.' const methodDetails = new Map([ - ['getAccessibilityTree', 'macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'], + ['getAccessibilityTree', 'On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'], ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again.'], ['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'], ]) diff --git a/packages/experimental/cua-macos/src/index.ts b/packages/experimental/cua-macos/src/index.ts index 77eb2e4a0e..eb794ea7e8 100644 --- a/packages/experimental/cua-macos/src/index.ts +++ b/packages/experimental/cua-macos/src/index.ts @@ -9,6 +9,7 @@ import { createPermissionParameters } from './permission-guide.ts' import { nativeHostArchitecture, prepareNativeExecutable, type SourceSigning } from './native-build.ts' import { createSharedNativeChannel } from './shared-native.ts' import { installHeldKeys } from './held-keys.ts' +import { AX_TRAVERSAL_MAX } from './ax-limits.ts' import { GUIDANCE } from './guidance.ts' export { GUIDANCE } from './guidance.ts' @@ -43,8 +44,8 @@ export interface Config extends CuaConfig { /** Validate shared operation bounds and macOS source signing before preparing the helper. */ export const Config = Schema.object({ ...CuaConfigFields, - maxElements: CuaConfigFields.maxElements.default(5000), - maxDepth: CuaConfigFields.maxDepth.default(50), + maxElements: CuaConfigFields.maxElements.max(AX_TRAVERSAL_MAX.maxElements).default(5000), + maxDepth: CuaConfigFields.maxDepth.max(AX_TRAVERSAL_MAX.maxDepth).default(50), permissionPollIntervalMs: Schema.number().step(1).min(100).max(2 ** 31 - 1).default(1000), sourceSigning: Schema.union([ Schema.const('ad-hoc'), diff --git a/packages/experimental/cua-macos/tests/loader-composition.spec.ts b/packages/experimental/cua-macos/tests/loader-composition.spec.ts index 45a5b096a5..724da1f131 100644 --- a/packages/experimental/cua-macos/tests/loader-composition.spec.ts +++ b/packages/experimental/cua-macos/tests/loader-composition.spec.ts @@ -173,13 +173,13 @@ it.skipIf(process.platform === 'win32')('shares one application between retained }) const retainedApp = await objectApi(agent).getApp('test.fixture') const retainedWindow = await retainedApp.getWindow(7) - await retainedApp.getAccessibilityTree({ maxDepth: 60 }) - expect(await retainedWindow.getAccessibilityTree({ maxElements: 4000, emit: false })).toContain('Count: 1') + await retainedApp.getAccessibilityTree({ maxDepth: 512 }) + expect(await retainedWindow.getAccessibilityTree({ maxElements: 100000, emit: false })).toContain('Count: 1') await retainedWindow.getAccessibilityTree({ emit: false }) const reads = (await readEvents()).filter(event => event.method === 'getState').slice(-3) expect(reads.map(event => event.params)).toEqual([ - { appHandle: retainedApp.appHandle, maxElements: 5000, maxDepth: 60 }, - { appHandle: retainedApp.appHandle, windowId: 7, maxElements: 4000, maxDepth: 50 }, + { appHandle: retainedApp.appHandle, maxElements: 5000, maxDepth: 512 }, + { appHandle: retainedApp.appHandle, windowId: 7, maxElements: 100000, maxDepth: 50 }, { appHandle: retainedApp.appHandle, windowId: 7, maxElements: 5000, maxDepth: 50 }, ]) const peer = await context.agents.create({ sessionId: SessionId('native-loader-peer'), agentOptions: { provider: 'native-fixture', model: 'vision' } }) diff --git a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts index b3830cbc9a..643f5ac4a2 100644 --- a/packages/experimental/cua-macos/tests/protocol-generation.spec.ts +++ b/packages/experimental/cua-macos/tests/protocol-generation.spec.ts @@ -21,6 +21,16 @@ describe('native protocol generation', () => { expect(validate({ id: 'invalid', method: 'setValue', params: target })).toBe(false) }) + it('enforces macOS tree budgets at native admission', () => { + const validate = new Ajv2020().compile(JSON.parse(renderNativeProtocol().json)) + const read = (maxElements: number, maxDepth: number) => ({ + id: 'tree', method: 'getState', params: { appHandle: 'app', maxElements, maxDepth }, + }) + expect(validate(read(100000, 512))).toBe(true) + expect(validate(read(100001, 512))).toBe(false) + expect(validate(read(100000, 513))).toBe(false) + }) + it('rejects a key array at both model and native pressKey admission', () => { const request = { method: 'pressKey', params: { appHandle: 'app', key: ['w', 'd'] } } expect(Request.safeParse(request).success).toBe(false) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index c610af3f03..b8641eebed 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -96,6 +96,10 @@ it('projects shared controls and macOS signing without adding signing to the sha } expect(CuaConfig({})).toMatchObject({ maxElements: 2000, maxDepth: 25 }) expect(NativeProvider.Config({})).toMatchObject({ maxElements: 5000, maxDepth: 50 }) + const limits = { maxElements: 100000, maxDepth: 512 } + expect(NativeProvider.Config(limits)).toMatchObject(limits) + expect(() => NativeProvider.Config({ maxElements: 100001 })).toThrow() + expect(() => NativeProvider.Config({ maxDepth: 513 })).toThrow() expect(CuaConfig({})).not.toHaveProperty('sourceSigning') expect(CuaConfig.dict).not.toHaveProperty('sourceSigning') expect(NativeProvider.Config({ sourceSigning: 'ad-hoc', timeoutMs: 1234 })).toMatchObject({ diff --git a/packages/experimental/cua/src/guidance.ts b/packages/experimental/cua/src/guidance.ts index 22f623649f..21791a7784 100644 --- a/packages/experimental/cua/src/guidance.ts +++ b/packages/experimental/cua/src/guidance.ts @@ -24,7 +24,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index c78008f14e..aca09ed0d3 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -7,7 +7,8 @@ const id = z.number().int().positive().max(2 ** 32 - 1) const pid = z.number().int().positive().max(2 ** 31 - 1) const point = z.number().nonnegative() const relativePoint = z.number().int().min(0).max(999) -const traversalLimits = z.object({ +const traversalLimit = z.number().min(1).multipleOf(1) +const nativeTraversalLimits = z.object({ maxElements: z.number().int().min(1).max(10000), maxDepth: z.number().int().min(1).max(100), }).strict() @@ -54,7 +55,7 @@ function requestSchema(hover: T) { operation('getState', { ...app, windowId: id.optional(), includeMenus: z.boolean().optional().describe('Expand menu contents; omitted or false shows menu-bar entries only.'), details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'), - ...traversalLimits.partial().shape, + maxElements: traversalLimit.optional(), maxDepth: traversalLimit.optional(), }), operation('listActions', snapshot), operation('performAction', { ...snapshot, action: z.string().min(1), argument: jsonValue.optional() }), @@ -98,10 +99,10 @@ export const SystemPointerRequest = requestSchema(z.union(MouseMoveTargets.map(t /** Native resource fields. The Host fills omitted observation limits from the provider configuration. */ export const NativeResourceParameters = { - getState: traversalLimits, + getState: nativeTraversalLimits, drag: z.object({ durationMs: z.number().int().min(1).max(5000), steps: z.number().int().min(1).max(240) }).strict(), paste: z.object({ maxClipboardBytes: z.number().int().min(1024), pasteReadTimeoutMs: z.number().int().min(1).max(2 ** 31 - 1), - ...traversalLimits.shape, + ...nativeTraversalLimits.shape, }).strict(), findApps: z.object({ maxAppMatches: z.number().int().min(1).max(1000) }).strict(), } diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index 8f37862aaf..d98897f5d4 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -38,9 +38,9 @@ export interface CuaObservationOptions { emit?: boolean } export interface CuaAxOptions extends CuaObservationOptions { /** Reserved compatibility option; observations are never diffs. */ disableDiffing?: boolean - /** Maximum nodes visited for this read, from 1 to 10000. Omission uses the provider setting. */ + /** Maximum nodes visited for this read. Omission uses the provider setting. */ maxElements?: number - /** Maximum traversal depth for this read, from 1 to 100. Omission uses the provider setting. */ + /** Maximum traversal depth for this read. Omission uses the provider setting. */ maxDepth?: number /** Expand menu contents. The default shows menu-bar entries only. */ includeMenus?: boolean diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index 437a6cb7df..1fa797cdd4 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -919,7 +919,7 @@ it('forwards bounded traversal limits on a window-only provider', async () => { const app = await f.api.getApp('Editor') const win = await app.getWindow(23) const before = f.dispatch.mock.calls.length - for (const limits of [{ maxDepth: 0 }, { maxDepth: 101 }, { maxElements: 1.5 }, { maxElements: 10001 }]) { + for (const limits of [{ maxDepth: 0 }, { maxElements: 1.5 }]) { await expect(win.getAccessibilityTree(limits)).rejects.toThrow(/maxDepth|maxElements/) } expect(f.dispatch).toHaveBeenCalledTimes(before) diff --git a/packages/experimental/cua/tests/protocol.spec.ts b/packages/experimental/cua/tests/protocol.spec.ts index 338d8e6752..c24ced44cd 100644 --- a/packages/experimental/cua/tests/protocol.spec.ts +++ b/packages/experimental/cua/tests/protocol.spec.ts @@ -6,7 +6,7 @@ import { CapabilitiesResult, Request, ScreenshotResult, SystemPointerRequest } f const rejectedRequests: Array<{ reason: string; request: JsonValue }> = [ { reason: 'raw process ID instead of a selected app handle', request: { method: 'typeText', params: { pid: 17, text: 'hello' } } }, { reason: 'unknown native method', request: { method: 'shutdown', params: {} } }, - { reason: 'node limit above the accepted range', request: { method: 'getState', params: { appHandle: 'app', maxElements: 1000000 } } }, + { reason: 'zero node budget', request: { method: 'getState', params: { appHandle: 'app', maxElements: 0 } } }, { reason: 'extra tool argument', request: { method: 'listApps', params: {}, helperPath: '/untrusted/helper' } }, { reason: 'element click without an observed snapshot', request: { method: 'click', params: { appHandle: 'app', elementId: '0' } } }, { reason: 'numeric element id', request: { method: 'click', params: { appHandle: 'app', snapshotId: 'snapshot', elementId: 0 } } }, @@ -67,10 +67,10 @@ it.each([ }) it('accepts disclosure options and bounded per-read traversal limits', () => { - const params = { appHandle: 'app', includeMenus: true, details: false, maxDepth: 100, maxElements: 10000 } + const params = { appHandle: 'app', includeMenus: true, details: false, maxDepth: 513, maxElements: 100001 } expect(Request.parse({ method: 'getState', params })).toEqual({ method: 'getState', params }) expect(Request.safeParse({ method: 'getState', params: { ...params, details: 'all' } }).success).toBe(false) - expect(Request.safeParse({ method: 'getState', params: { ...params, maxDepth: 101 } }).success).toBe(false) + expect(Request.safeParse({ method: 'getState', params: { ...params, maxDepth: -1 } }).success).toBe(false) }) it.each([ From 9bd4a9aad7210c5dcd4aa9f8a7bfa9bb6d0e623c Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 14:13:11 +0800 Subject: [PATCH 04/10] test(cua-macos): exercise traversal budgets on the AppKit fixture --- packages/experimental/cua-macos/native/tests/e2e.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/experimental/cua-macos/native/tests/e2e.py b/packages/experimental/cua-macos/native/tests/e2e.py index 5eeeaac59b..83782db1a2 100644 --- a/packages/experimental/cua-macos/native/tests/e2e.py +++ b/packages/experimental/cua-macos/native/tests/e2e.py @@ -167,7 +167,9 @@ def main(): time.sleep(0.025) primary_params = {**owner, "windowId": primary["windowId"], "maxElements": 300, "maxDepth": 20} secondary_params = {**owner, "windowId": secondary["windowId"], "maxElements": 300, "maxDepth": 20} - first = call("getState", primary_params) + limited = call("getState", {**primary_params, "maxElements": 1}) + assert limited["truncated"] and "maxElements=1" in limited["tree"], limited + first = call("getState", {**primary_params, "maxElements": 100000, "maxDepth": 512}) second = call("getState", secondary_params) assert "Increment secondary" not in first["tree"], first assert "Increment primary" not in second["tree"], second From dfc8671279d7d8a143cc79c306d6389cb725b15e Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 14:14:21 +0800 Subject: [PATCH 05/10] test(cua): refresh portable AX option and macOS guidance snapshots --- snapshots/session/cua-native/system-prompt.expected.md | 2 +- snapshots/session/cua-native/tool-schemas.expected.json | 8 ++++---- snapshots/session/cua-ptc/system-prompt.expected.md | 2 +- .../session/ptc-script-retained/system-prompt.expected.md | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 74b65450b8..8de11e7f91 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -57,7 +57,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index ed09a6bb40..eeeddd5e6f 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -306,14 +306,14 @@ "type": "boolean" }, "maxElements": { - "type": "integer", + "type": "number", "minimum": 1, - "maximum": 10000 + "multipleOf": 1 }, "maxDepth": { - "type": "integer", + "type": "number", "minimum": 1, - "maximum": 100 + "multipleOf": 1 } }, "required": [ diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index e880caf155..c024d5eb82 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -59,7 +59,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index fadc73733b..94583f8ff7 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -59,7 +59,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — override the node limit (1 to 10000) or depth limit (1 to 100) for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS defaults to 5000 nodes and depth 50. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. From 1bd41d063ecedb0ccdf87886e01175ee8e2213b6 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 14:17:42 +0800 Subject: [PATCH 06/10] docs(cua): record why AX configuration sets per-read defaults --- ...-11-09-cua-ax-traversal-defaults.i18n.yaml | 19 +++++++++++++++++ .../2026-11-09-cua-ax-traversal-defaults.md | 21 +++++++++++++++++++ ...2026-11-09-cua-ax-traversal-defaults.zh.md | 21 +++++++++++++++++++ docs/config-catalog.md | 2 +- docs/config-catalog.zh.md | 2 +- .../experimental/cua-macos/README.i18n.yaml | 4 ++-- packages/experimental/cua-macos/README.md | 2 +- packages/experimental/cua-macos/README.zh.md | 2 +- packages/experimental/cua/src/index.ts | 4 ++-- 9 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 .agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.i18n.yaml create mode 100644 .agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.md create mode 100644 .agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.zh.md diff --git a/.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.i18n.yaml b/.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.i18n.yaml new file mode 100644 index 0000000000..0e2260eefc --- /dev/null +++ b/.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.i18n.yaml @@ -0,0 +1,19 @@ +# Bilingual-pair consistency record for 2026-11-09-cua-ax-traversal-defaults.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.md +/agent-note-ax-traversal-defaults-and-per-read-limits: + en: 71a088b589135b94 + zh: 40a0accb1f6d100b +/agent-note-ax-traversal-defaults-and-per-read-limits/problem: + en: 18405553f792f64c + zh: 1b6891403539c2f8 +/agent-note-ax-traversal-defaults-and-per-read-limits/decision: + en: 89cb2b1f33b0551c + zh: b57839093a735c2a +/agent-note-ax-traversal-defaults-and-per-read-limits/alternatives-considered: + en: c15d84e0bbb18b39 + zh: 4799291d7a66fa04 +/agent-note-ax-traversal-defaults-and-per-read-limits/consequences: + en: 687f4ba2af13e847 + zh: ae5777022619b71d diff --git a/.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.md b/.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.md new file mode 100644 index 0000000000..37a04b4aa7 --- /dev/null +++ b/.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.md @@ -0,0 +1,21 @@ +# Agent Note: AX traversal defaults and per-read limits + +Status: implemented + +English | [中文](2026-11-09-cua-ax-traversal-defaults.zh.md) + +## Problem + +A shallow read can omit a control in a deeply nested window. Increasing the profile settings to reach it also increases the work and possible output of subsequent reads. + +## Decision + +For an accessibility tree read, the Host takes each of `maxElements` and `maxDepth` from the request when present and from the provider configuration otherwise. A caller can raise or lower either value for one read. The native provider enforces its traversal ceilings. On macOS, Paste menu discovery always uses the configured values because the paste operation has no traversal options. + +## Alternatives considered + +**Cap each read at the configured values.** With a profile tuned for small reads, recovering a truncated branch would require changing the profile even when the provider can handle a larger request. Per-read overrides let the caller inspect that window and return to the usual limits on the next read. + +## Consequences + +Lowering a profile default reduces the budget for ordinary reads. An explicit larger request can take longer and produce more text. The native operation deadline and response size limit still apply. The [macOS configuration table](../../../../packages/experimental/cua-macos/README.md#use-this-package) lists the defaults. diff --git a/.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.zh.md b/.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.zh.md new file mode 100644 index 0000000000..536a6908eb --- /dev/null +++ b/.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.zh.md @@ -0,0 +1,21 @@ +# Agent Note: AX 树遍历的默认值和单次读取上限 + +Status: implemented + +[English](2026-11-09-cua-ax-traversal-defaults.md) | 中文 + +## 问题 + +读取深度较小时，控件可能位于窗口中被截断的分支里。为了找到这个控件而调高配置值，会让以后的读取也可能访问更多节点、返回更多文本。 + +## 决策 + +读取辅助功能树时，Host 分别处理 `maxElements` 和 `maxDepth`。调用方显式传入的值只对本次读取生效，可以高于或低于配置值。省略的参数使用平台配置。原生实现在遍历时执行自身的硬上限。macOS 的粘贴操作没有遍历参数，查找粘贴菜单时始终使用配置值。 + +## 考虑过的替代方案 + +**把配置值作为每次读取的硬上限。** 如果日常读取使用较小的配置值，即使平台支持读取更大的树，调用方也必须先修改配置才能看到被截断的分支。允许单次覆盖后，调用方可以单独扩大这次读取的范围，下次读取继续使用原有配置。 + +## 影响 + +调低配置值可以缩小常规读取的范围。显式请求更大的树可能耗时更长，也可能返回更多文本。原生调用仍受超时和响应大小限制。默认值见 [macOS 配置表](../../../../packages/experimental/cua-macos/README.zh.md#use-this-package)。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 73bf93e599..00eafb403c 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1156,7 +1156,7 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:34`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 06b6de880b..011611902b 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -1158,7 +1158,7 @@ export interface Config { - `inject`: `computerUse` · `tools` · `systemPrompt` · `subprocess` · `agents` - `refs`: [`CuaConfig`](../packages/experimental/cua/src/index.ts) -- `source`: [`packages/experimental/cua-macos/src/index.ts:33`](../packages/experimental/cua-macos/src/index.ts) +- `source`: [`packages/experimental/cua-macos/src/index.ts:34`](../packages/experimental/cua-macos/src/index.ts) ```ts config-catalog /** Native preparation, source-build signing, operation bounds, and profile-local application grants. */ diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index e26f3129a0..568fc1f202 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -15,8 +15,8 @@ en: 1777e7bf6785f986 zh: e3daac1074ebb74d /deepseek-ai-dsh-experimental-cua-macos/use-this-package: - en: e25eb1f82970b346 - zh: 418e6d1a9775948d + en: 7ff2441bd33b11b0 + zh: 7811c00c9d828a86 /deepseek-ai-dsh-experimental-cua-macos/use-this-package/connect-an-external-execution-environment: en: 652b0ce8892e522c zh: 101c20aa18b36c05 diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index aeb34b100b..c30a2b0362 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -72,7 +72,7 @@ Source builds default to `sourceSigning: ad-hoc`, without creating certificates | `maxClipboardBytes` | `33554432` | Maximum original clipboard bytes preserved during a paste; oversized contents reject before replacement. | | `pasteReadTimeoutMs` | `1000` | Clipboard read deadline after the Paste command returns; expiry reports `PASTE_NOT_CONSUMED`. | | `maxAppMatches` | `20` | Maximum installed-app search matches; broader queries reject and require a narrower name. | -| `maxElements` / `maxDepth` | `5000` / `50` | Accessibility traversal limits. | +| `maxElements` / `maxDepth` | `5000` / `50` | Defaults for tree reads. Each read can raise or lower either value. Paste menu discovery always uses the configured limits. [Rationale](../../../.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.md) | | `maxClickHoldMs` | `5000` | Maximum explicit click hold in milliseconds; accepts nonnegative integers. Zero only permits immediate release. | | `dragDurationMs` / `dragSteps` | `400` / `20` | Duration and movement segments for a drag. | | `textEventIntervalMs` | `0.25` | Delay after each Unicode down/up event, from 0 to 999 ms. Keystroke timing is independent. | diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index c6ed84e59c..89286b19c6 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -72,7 +72,7 @@ pnpm dsh headless --patch packages/experimental/cua-macos/examples/macos.cordis. | `maxClipboardBytes` | `33554432` | 粘贴时保存原剪贴板内容的字节上限；超限会在替换前拒绝。 | | `pasteReadTimeoutMs` | `1000` | Paste 命令返回后等待剪贴板读取的上限；到期返回 `PASTE_NOT_CONSUMED`。 | | `maxAppMatches` | `20` | 已安装应用搜索的结果上限；范围过大的查询会被拒绝，需要缩小名称范围。 | -| `maxElements` / `maxDepth` | `5000` / `50` | 辅助功能树遍历限制。 | +| `maxElements` / `maxDepth` | `5000` / `50` | 读取树时的默认值。每次读取可单独调高或调低任一项。查找粘贴菜单始终使用配置值。[决策理由](../../../.agents/notes/implemented/architecture/2026-11-09-cua-ax-traversal-defaults.zh.md) | | `maxClickHoldMs` | `5000` | 显式点击按住时长的上限，单位为毫秒，接受非负整数。设为 0 时只允许立即松开。 | | `dragDurationMs` / `dragSteps` | `400` / `20` | 拖动时长和移动分段数。 | | `textEventIntervalMs` | `0.25` | 每个 Unicode 按下或抬起事件后的间隔，范围为 0 至 1000 毫秒。击键节奏独立。 | diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index e4f3ec8313..eecc8cc90c 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -77,9 +77,9 @@ export interface CuaConfig { textEventIntervalMs: number /** Native connection cleanup, socket-write and launcher termination budget in milliseconds. */ graceMs: number - /** Default node limit for accessibility reads and Paste menu discovery. */ + /** Default node limit for accessibility reads and fixed node limit for Paste menu discovery. */ maxElements: number - /** Default depth limit for accessibility reads and Paste menu discovery. */ + /** Default depth limit for accessibility reads and fixed depth limit for Paste menu discovery. */ maxDepth: number } From 9425b6defe6773b94ced7fd4defba2dad8583703 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 14:23:46 +0800 Subject: [PATCH 07/10] fix(cua): report configured AX budgets and document migration --- .../cua-ax-read-limits/guide.i18n.yaml | 16 +++++++++++++++ .../v0.2.2-rc.1/cua-ax-read-limits/guide.md | 20 +++++++++++++++++++ .../cua-ax-read-limits/guide.zh.md | 20 +++++++++++++++++++ .../cua-native-platform/guide.i18n.yaml | 4 ++-- .../v0.2.2-rc.1/cua-native-platform/guide.md | 2 +- .../cua-native-platform/guide.zh.md | 2 +- .../experimental/cua-macos/README.i18n.yaml | 8 ++++---- packages/experimental/cua-macos/README.md | 6 ++++-- packages/experimental/cua-macos/README.zh.md | 6 ++++-- .../experimental/cua-macos/src/guidance.ts | 2 +- .../cua-macos/tests/guidance.spec.ts | 2 +- .../tests/provider-lifecycle.spec.ts | 1 + packages/experimental/cua/src/index.ts | 6 +++--- packages/experimental/cua/src/protocol.ts | 5 +++-- packages/experimental/cua/src/types.ts | 8 ++++---- packages/experimental/cua/tests/api.spec.ts | 5 ++--- .../cua-native/workspace/native-helper.mjs | 3 +++ .../session/cua-ptc/replay.override.json | 2 +- 18 files changed, 91 insertions(+), 27 deletions(-) create mode 100644 docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.i18n.yaml create mode 100644 docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.md create mode 100644 docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.zh.md diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.i18n.yaml new file mode 100644 index 0000000000..aa355c3d62 --- /dev/null +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.i18n.yaml @@ -0,0 +1,16 @@ +# Bilingual-pair consistency record for guide.md (docs/i18n/README.md): per heading +# section, a hash of its English and Chinese blocks outside code blocks and generated regions. +# After editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.md +/: + en: a8ca814cf60ea12e + zh: f068d1ee13e80594 +/macos-ax-tree-read-limits: + en: a7eed0c191ff2b32 + zh: 15291798ba91329b +/macos-ax-tree-read-limits/change: + en: e4281d86095b950f + zh: 684ddf2957443ace +/macos-ax-tree-read-limits/migration: + en: 97a3885f5d1cb907 + zh: 19313c56eff68835 diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.md new file mode 100644 index 0000000000..8eab347bb2 --- /dev/null +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.md @@ -0,0 +1,20 @@ +--- +kind: upgrade-guide +description: "macOS reads more AX nodes by default and accepts per-read overrides of configured traversal budgets." +--- + +# macOS AX tree read limits + +English | [中文](guide.zh.md) + +## Change + +The default macOS node count rises from 2000 to 5000 and the default depth from 25 to 50. Reading a large window can take longer and emit more text to the model. + +Configured `maxElements` and `maxDepth` now supply defaults independently for omitted arguments. Callers can request larger or smaller values for one read. Paste menu discovery continues to use the configured limits. + +## Migration + +1. To keep the previous budgets for ordinary reads, find the `@deepseek-ai/dsh-experimental-cua-macos` entry in your `cordis.yml` or profile patch. Set `maxElements: 2000` and `maxDepth: 25` in its existing `config`. +2. Restart the profile and confirm both defaults in the Computer Use model instructions. +3. If a script depends on specific traversal budgets, pass them explicitly to `getAccessibilityTree()`. An override applies only to that call. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.zh.md new file mode 100644 index 0000000000..e6f392c7aa --- /dev/null +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-ax-read-limits/guide.zh.md @@ -0,0 +1,20 @@ +--- +kind: upgrade-guide +description: "macOS 默认读取更多 AX 节点，调用方可为单次读取覆盖配置值。" +--- + +# macOS AX 树读取范围 + +[English](guide.md) | 中文 + +## 变更 + +macOS 的默认节点数从 2000 增至 5000，默认深度从 25 增至 50。窗口中有更多节点可访问时，读取可能耗时更长，输出到模型的文本也可能增加。 + +`maxElements` 和 `maxDepth` 分别决定省略对应参数时的读取范围。调用方可以为一次读取指定高于或低于配置的值。查找粘贴菜单始终使用配置值。 + +## 迁移 + +1. 如需沿用原先的常规读取范围，在 `cordis.yml` 或配置补丁中找到 `@deepseek-ai/dsh-experimental-cua-macos`，在现有的 `config` 下设置 `maxElements: 2000` 和 `maxDepth: 25`。 +2. 重启配置，确认 Computer Use 模型说明中的两个默认值与配置一致。 +3. 如果脚本依赖特定的遍历范围，在 `getAccessibilityTree()` 的调用处显式传入这两个参数。一次调用的覆盖值不会影响后续读取。 diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml index a017e7f82b..6dfabdc004 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.i18n.yaml @@ -12,5 +12,5 @@ en: e5d1027faa55adba zh: 4b88ad6a70977fb3 /native-cua-application-and-window-api/migration: - en: a63e88dce467d6ee - zh: fc25bb283ca0eaff + en: 106eab81fd078fec + zh: 6fb30738af8bc621 diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md index 2675699a71..2f4eea3c91 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.md @@ -16,7 +16,7 @@ Providers supply startup capabilities alongside their platform guidance. The nat ## Migration 1. After `cua.getApp(selector)`, select the target from `app.getWindows()` with `app.getWindow(windowId)`. Use `window.getScreenshot()`, `window.click()`, `window.scroll()`, and `window.drag()`. Replace `hover` with `window.mouseMove()`. On macOS, `window.click(id)` sends mouse events at the element center. Switch to `invoke(id)` for the default accessibility action, or `performAction(id, "AXShowMenu")` to request an advertised menu action. -2. Replace `getAXState()` with `getAccessibilityTree()`. Replace `getAXStateAndScreenshot()` with separate tree and screenshot reads. Selection reads the initial tree silently for a window, and for an app when `appLevelAccessibility` is true. Read the tree explicitly to display it. +2. Replace `getAXState()` with `getAccessibilityTree()`. Replace `getAXStateAndScreenshot()` with separate tree and screenshot reads. Selection reads the initial tree silently for a window, and for an app when `appLevelAccessibility` is true. Read the tree explicitly to display it. See [macOS AX read limits](../cua-ax-read-limits/guide.md) for changes to traversal defaults and configuration. 3. Pass the opaque string ID from the latest tree wherever you passed a numeric element index. In raw requests use `elementId` with `snapshotId`. Replace `performSecondaryAction` with `listActions(id)` and `performAction(id, exactName, argument?)`, or use `invoke(id)` for the default action. `setValue` accepts the provider’s supported JSON values. On macOS, `performAction(id, 'AXRaise')` performs the advertised raise action. Call `app.activate()` first when the application must become active. 4. Replace a packed chord such as `pressKey('shift+Tab')` with `pressKey('Tab', ['shift'])`. Check `(await cua.getCapabilities()).methods` before using `sendKeyEvent`. On macOS, call it inside one `run_code` to hold a key across tool calls. To request one repeat, send another down for a non-modifier key held by that execution. Returning, throwing or pausing releases any remaining keys. If your script handles unknown macOS key names as `NATIVE_ERROR`, handle `INVALID_ARGUMENT` instead. Correct the key name before retrying. 5. Read `dispatched` from action results, then observe the application to verify the outcome. `invoke` returns `dispatched: false` when the element has no default action. Remove reads of the old `verified` field and of `pid` on selected objects or window records. Use `listApps()` or running `findApps()` matches when you need a PID. diff --git a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md index b4551f624b..77e65e7ba4 100644 --- a/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md +++ b/docs/upgrade-guide/v0.2.2-rc.1/cua-native-platform/guide.zh.md @@ -16,7 +16,7 @@ CUA 脚本在指定窗口上截图和操作鼠标。App 和 Window 对象共享 ## 迁移 1. 调用 `cua.getApp(selector)` 后，从 `app.getWindows()` 返回的列表选定目标，再调用 `app.getWindow(windowId)`。在窗口上调用 `getScreenshot()`、`click()`、`scroll()` 和 `drag()`，将原有的 `hover` 调用改为 `window.mouseMove()`。在 macOS 上，`window.click(id)` 在元素中心发送鼠标事件。需要执行默认辅助功能动作时，改用 `invoke(id)`。需要打开控件菜单且控件声明了 `AXShowMenu` 时，调用 `performAction(id, "AXShowMenu")`。 -2. 将 `getAXState()` 改为 `getAccessibilityTree()`，将 `getAXStateAndScreenshot()` 拆成读取树和截图两次调用。选择窗口时会静默读取初始树。选择应用时，只有 `appLevelAccessibility` 为真才读取。要显示树，请显式读取。 +2. 将 `getAXState()` 改为 `getAccessibilityTree()`，将 `getAXStateAndScreenshot()` 拆成读取树和截图两次调用。选择窗口时会静默读取初始树。选择应用时，只有 `appLevelAccessibility` 为真才读取。要显示树，请显式读取。[读取上限的迁移说明](../cua-ax-read-limits/guide.zh.md)列出了 macOS 默认值和配置含义的变化。 3. 将数字元素索引改为从最新树中取得的字符串 ID。直接发送请求时使用 `elementId` 和 `snapshotId`。原先调用 `performSecondaryAction` 的地方，先用 `listActions(id)` 取得完整动作名，再调用 `performAction(id, exactName, argument?)`。要执行默认动作，可调用 `invoke(id)`。`setValue` 接受提供方支持的 JSON 值。 在 macOS 上，`performAction(id, 'AXRaise')` 只执行元素声明的抬起窗口动作。需要把应用切到前台时先调用 `app.activate()`。 4. 将 `pressKey('shift+Tab')` 之类的组合键写法改为 `pressKey('Tab', ['shift'])`。调用 `sendKeyEvent` 前检查 `(await cua.getCapabilities()).methods`。在 macOS 上，同一次 `run_code` 中可调用此方法跨工具调用保持按键。如果需要一次重复按下，可在这次执行中对已按住的普通键再发一次 `down`。执行返回、异常退出或用户暂停时会释放仍按着的键。如果脚本用 `NATIVE_ERROR` 识别 macOS 上的未知键名，请改为处理 `INVALID_ARGUMENT`，并在重试前修正键名。 5. 从动作结果读取 `dispatched`，再观察应用确认效果。元素没有默认动作时，`invoke` 返回 `dispatched: false`。移除读取旧字段 `verified` 的代码，以及对已选对象和窗口记录中 `pid` 的读取。需要 PID 时，使用 `listApps()` 或 `findApps()` 返回的运行中应用记录。 diff --git a/packages/experimental/cua-macos/README.i18n.yaml b/packages/experimental/cua-macos/README.i18n.yaml index 568fc1f202..a9cd9503ed 100644 --- a/packages/experimental/cua-macos/README.i18n.yaml +++ b/packages/experimental/cua-macos/README.i18n.yaml @@ -39,14 +39,14 @@ en: 2dbc22ffd1f8a717 zh: d4458872e75305d9 /deepseek-ai-dsh-experimental-cua-macos/model-experience/native-application-operations/token-effect: - en: ab52b46d874fa8b9 - zh: e87d3620ef77cb06 + en: f49bbb1f447feb1d + zh: 664edf0f704caab6 /deepseek-ai-dsh-experimental-cua-macos/model-experience/native-application-operations/kv-cache-effect: en: a2c9acb5a4320dda zh: e8d1586af60215c4 /deepseek-ai-dsh-experimental-cua-macos/known-limitations-and-deferred-work: - en: 4fefaf88ea9eefc6 - zh: d3d18cfbccf8c20f + en: b362f9357fe79293 + zh: a74f0e9cffb647bc /deepseek-ai-dsh-experimental-cua-macos/known-limitations-and-deferred-work/dev-note: en: b836f01bd0bceb4a zh: 8122fa08b2aa95fa diff --git a/packages/experimental/cua-macos/README.md b/packages/experimental/cua-macos/README.md index c30a2b0362..7f2a0ac6c2 100644 --- a/packages/experimental/cua-macos/README.md +++ b/packages/experimental/cua-macos/README.md @@ -202,7 +202,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS accepts maxElements up to 100000 and maxDepth up to 512. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. @@ -233,7 +233,7 @@ Only if the task cannot be completed without moving the user’s pointer, call a #### Token effect -Enabling the plugin adds a fixed schema and guidance. AX observations, operation results and admitted images add model input. Retained objects are not serialized into prompts by this package. +Enabling the plugin adds a fixed schema and guidance. AX observations, operation results and admitted images add model input. Retained objects are not serialized into prompts by this package. A higher default or per-read override can include more AX text in model input when the window has more nodes to visit. #### KV Cache effect @@ -241,6 +241,8 @@ Tool results append to Session history. Changing plugin configuration or tool pr ## Known Limitations and Deferred Work +The native operation deadline (`timeoutMs`), per-AX-call timeout (`axTimeoutMs`), and response size limit (`maxResponseBytes`) still apply to larger tree reads. + The queue orders individual native requests. Holding an app object does not reserve its application across a Session; [continuous interaction](../../../.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.md) grants no cross-call ownership. Capture RPCs that never return occupy the single capture slot; creating new app objects does not reset that system connection. diff --git a/packages/experimental/cua-macos/README.zh.md b/packages/experimental/cua-macos/README.zh.md index 89286b19c6..db919ff43c 100644 --- a/packages/experimental/cua-macos/README.zh.md +++ b/packages/experimental/cua-macos/README.zh.md @@ -202,7 +202,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS accepts maxElements up to 100000 and maxDepth up to 512. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. @@ -233,7 +233,7 @@ Only if the task cannot be completed without moving the user’s pointer, call a #### Token 影响 -启用插件会增加固定参数定义和指导文本。AX 观察、操作结果和已接入的图片会增加模型输入。本包不将保留的对象序列化进提示。 +启用插件会增加固定参数定义和指导文本。AX 观察、操作结果和已接入的图片会增加模型输入。本包不将保留的对象序列化进提示。窗口中有更多节点可读时，调高默认值或单次读取上限可能使进入模型的 AX 文本变长。 #### KV Cache 影响 @@ -241,6 +241,8 @@ Only if the task cannot be completed without moving the user’s pointer, call a ## 已知限制与后续工作 +读取较大的树仍受原生操作超时（`timeoutMs`）、单次 AX 调用超时（`axTimeoutMs`）和响应大小限制（`maxResponseBytes`）约束。 + 队列只排序单次原生请求。持有 app 对象不会为 Session 保留应用，[连续交互](../../../.agents/notes/implemented/architecture/2026-10-13-cua-continuous-hover.zh.md)不分配跨调用所有权。一直不返回的系统截图 RPC 会占用唯一捕获槽位，重新创建 app 对象不会重置该系统连接。 diff --git a/packages/experimental/cua-macos/src/guidance.ts b/packages/experimental/cua-macos/src/guidance.ts index 1ed115743f..e91d4d391e 100644 --- a/packages/experimental/cua-macos/src/guidance.ts +++ b/packages/experimental/cua-macos/src/guidance.ts @@ -5,7 +5,7 @@ import { COMMON_GUIDANCE } from '@deepseek-ai/dsh-cua/guidance' export const PLATFORM_GUIDANCE = 'macOS: accessibility tree operations are available at app or window level. Simulated keyboard input is available at app level. Simulated mouse input is available at window level. For pointer input on a sheet, select its window ID with app.getWindow(id). For menu items outside a window, use app.invoke(id). findApps searches /Applications, /System/Applications, and ~/Applications. Use the Computer Use permission page to grant the helper application Accessibility and Screen Recording permissions in System Settings. Native window and AX bounds use macOS points, while delivered screenshots carry their own pixel dimensions. show does not switch Spaces, and Stage Manager can present reduced window geometry; explain any required foreground step before calling app.activate() or performing an advertised AXRaise action.' const methodDetails = new Map([ - ['getAccessibilityTree', 'On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'], + ['getAccessibilityTree', 'macOS accepts maxElements up to 100000 and maxDepth up to 512. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current.'], ['mouseMove', 'Default hover requires both AXShowAlternateUI and AXShowDefaultUI on an enabled element or ancestor. Without that route, HOVER_UNAVAILABLE is returned before sending input. Physical movement requires an unobscured foreground target. Explain any necessary activation and observe the target again.'], ['click', 'click is a Window method: select a window with const win = await app.getWindow(windowId), then call win.click(target). For a background control’s default action, use app.invoke(id). For a mouse click on a background app, call app.activate() first and re-read the target. Both target forms send mouse-down and mouse-up events. click(id) uses the center of the full element bounds. If that center is outside the window, take a fresh win.getScreenshot() and use click([x, y]) on a visible part of the control in screenshot pixels. A background app can ignore the events even when dispatched is true. Verify the expected effect, such as a changed selection, before typing or retrying.'], ]) diff --git a/packages/experimental/cua-macos/tests/guidance.spec.ts b/packages/experimental/cua-macos/tests/guidance.spec.ts index 54d384a688..55774b8772 100644 --- a/packages/experimental/cua-macos/tests/guidance.spec.ts +++ b/packages/experimental/cua-macos/tests/guidance.spec.ts @@ -63,7 +63,7 @@ it('enumerates every available method once in its catalog entry', () => { }) it('documents option fields and return fields', () => { - for (const token of ['emit', 'includeMenus', 'details', 'relative', 'mouseButton', 'clickCount', 'holdMs', 'modifiers', 'format', 'prefix', 'suffix', 'selectionType']) expect(COMMON_GUIDANCE).toContain(token) + for (const token of ['emit', 'includeMenus', 'details', 'maxElements', 'maxDepth', 'relative', 'mouseButton', 'clickCount', 'holdMs', 'modifiers', 'format', 'prefix', 'suffix', 'selectionType']) expect(COMMON_GUIDANCE).toContain(token) for (const token of ['dispatched', 'foregroundChanged', 'hoverRoute', 'systemPointerMoved', 'bytes', 'mimeType', 'pixelWidth', 'pixelHeight', 'windowId']) expect(COMMON_GUIDANCE).toContain(token) }) diff --git a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts index b8641eebed..19a062001b 100644 --- a/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua-macos/tests/provider-lifecycle.spec.ts @@ -774,6 +774,7 @@ it('prepares the selected native component before registering CUA and creates no it('adds an installation-owned bridge URL when persistent PTC is available', async () => { const f = await setup() const text = async () => (await f.ctx.systemPrompt.assemble()).sections.find(section => section.name === 'computer-use:dsh-cua')?.text + expect(await text()).toContain('AX read defaults: maxElements=64, maxDepth=8.') expect(await text()).not.toContain('Persistent PTC initialization:') await f.ctx.plugin(FileSystem) await f.ctx.plugin(Sandbox) diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index eecc8cc90c..e1054b80ba 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -77,9 +77,9 @@ export interface CuaConfig { textEventIntervalMs: number /** Native connection cleanup, socket-write and launcher termination budget in milliseconds. */ graceMs: number - /** Default node limit for accessibility reads and fixed node limit for Paste menu discovery. */ + /** Default node limit when an accessibility read omits maxElements. */ maxElements: number - /** Default depth limit for accessibility reads and fixed depth limit for Paste menu discovery. */ + /** Default depth limit when an accessibility read omits maxDepth. */ maxDepth: number } @@ -155,7 +155,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) const mouseControl = systemPointer ? new MouseControl(ctx, config.requireSystemPointerApproval) : undefined const requestSchema = systemPointer ? SystemPointerRequest : Request const instructions = systemPointer ? withSystemPointerGuidance(backend.guidance) : backend.guidance - const guidance = `${instructions}\n\nMaximum click holdMs: ${config.maxClickHoldMs} ms.` + const guidance = `${instructions}\n\nAX read defaults: maxElements=${config.maxElements}, maxDepth=${config.maxDepth}.\nMaximum click holdMs: ${config.maxClickHoldMs} ms.` const native: NativeClientConfig = { helperPath, cwd: process.cwd(), timeoutMs: config.timeoutMs, axTimeoutMs: config.axTimeoutMs, connectTimeoutMs: config.connectTimeoutMs, screenshotTimeoutMs: config.screenshotTimeoutMs, diff --git a/packages/experimental/cua/src/protocol.ts b/packages/experimental/cua/src/protocol.ts index aca09ed0d3..18d89b3919 100644 --- a/packages/experimental/cua/src/protocol.ts +++ b/packages/experimental/cua/src/protocol.ts @@ -55,7 +55,8 @@ function requestSchema(hover: T) { operation('getState', { ...app, windowId: id.optional(), includeMenus: z.boolean().optional().describe('Expand menu contents; omitted or false shows menu-bar entries only.'), details: z.boolean().optional().describe('Include identifiers, global native bounds and complete action names.'), - maxElements: traversalLimit.optional(), maxDepth: traversalLimit.optional(), + maxElements: traversalLimit.optional().describe('Maximum nodes to visit on this read. Omission uses the provider setting.'), + maxDepth: traversalLimit.optional().describe('Maximum depth to traverse on this read. Omission uses the provider setting.'), }), operation('listActions', snapshot), operation('performAction', { ...snapshot, action: z.string().min(1), argument: jsonValue.optional() }), @@ -97,7 +98,7 @@ export const SystemPointerRequest = requestSchema(z.union(MouseMoveTargets.map(t useSystemPointer: z.boolean().optional().describe('Request use of the system pointer. The Host checks permission before dispatch.'), })))) -/** Native resource fields. The Host fills omitted observation limits from the provider configuration. */ +/** Native resource fields supplied or completed by the Host before dispatch. */ export const NativeResourceParameters = { getState: nativeTraversalLimits, drag: z.object({ durationMs: z.number().int().min(1).max(5000), steps: z.number().int().min(1).max(240) }).strict(), diff --git a/packages/experimental/cua/src/types.ts b/packages/experimental/cua/src/types.ts index d98897f5d4..46ab4c26bd 100644 --- a/packages/experimental/cua/src/types.ts +++ b/packages/experimental/cua/src/types.ts @@ -38,14 +38,14 @@ export interface CuaObservationOptions { emit?: boolean } export interface CuaAxOptions extends CuaObservationOptions { /** Reserved compatibility option; observations are never diffs. */ disableDiffing?: boolean - /** Maximum nodes visited for this read. Omission uses the provider setting. */ - maxElements?: number - /** Maximum traversal depth for this read. Omission uses the provider setting. */ - maxDepth?: number /** Expand menu contents. The default shows menu-bar entries only. */ includeMenus?: boolean /** Include identifiers, global native bounds and complete action names. */ details?: boolean + /** Maximum nodes visited for this read. Omission uses the provider setting. */ + maxElements?: number + /** Maximum traversal depth for this read. Omission uses the provider setting. */ + maxDepth?: number } /** Reported permissions and operations; null sessionLocked means no usable console session was verified. */ export type CuaCapabilities = z.output diff --git a/packages/experimental/cua/tests/api.spec.ts b/packages/experimental/cua/tests/api.spec.ts index 1fa797cdd4..d2ffb89dfd 100644 --- a/packages/experimental/cua/tests/api.spec.ts +++ b/packages/experimental/cua/tests/api.spec.ts @@ -919,9 +919,8 @@ it('forwards bounded traversal limits on a window-only provider', async () => { const app = await f.api.getApp('Editor') const win = await app.getWindow(23) const before = f.dispatch.mock.calls.length - for (const limits of [{ maxDepth: 0 }, { maxElements: 1.5 }]) { - await expect(win.getAccessibilityTree(limits)).rejects.toThrow(/maxDepth|maxElements/) - } + await expect(win.getAccessibilityTree({ maxDepth: 0 })).rejects.toThrow('maxDepth') + await expect(win.getAccessibilityTree({ maxElements: 1.5 })).rejects.toThrow('maxElements') expect(f.dispatch).toHaveBeenCalledTimes(before) await win.getAccessibilityTree({ maxDepth: 1, maxElements: 1 }) expect(f.dispatch).toHaveBeenLastCalledWith({ diff --git a/snapshots/session/cua-native/workspace/native-helper.mjs b/snapshots/session/cua-native/workspace/native-helper.mjs index e9a972a7eb..c6c883bfd6 100755 --- a/snapshots/session/cua-native/workspace/native-helper.mjs +++ b/snapshots/session/cua-native/workspace/native-helper.mjs @@ -170,6 +170,9 @@ function createApplicationState(socket, record, application, captureBroker, conn snapshots.set(params.windowId ?? null, { snapshotId, windowId: params.windowId ?? null, axless }) if (axless) return { appHandle: app.appHandle, windowId: 8, snapshotId, tree: '', elements: [], windows: [axlessWindow], truncated: false } + if (params.maxElements === 1) return { appHandle: app.appHandle, windowId: params.windowId ?? null, snapshotId, + tree: `AX scope=${params.windowId === undefined ? 'application' : 'window'} offsets=UTF-16\n- [0] AXButton "Increment"\nAccessibility tree truncated (maxElements=1).`, + elements: [], windows: [window], truncated: true } return { appHandle: app.appHandle, windowId: params.windowId ?? null, snapshotId: `${scope}-snapshot-${generation}`, tree: `AX scope=${params.windowId === undefined ? 'application' : 'window'} offsets=UTF-16${params.details ? ' bounds=global-points' : ''}\n- [0] AXButton "Increment"${params.details ? ' id="increment" actions=["AXPress"] bounds=[60,80,100,20]' : ''}\n- [1] AXTextField valueWritable=true selectionWritable=true value=${JSON.stringify(application.value)} placeholder="Fixture text"\n- AXStaticText "Count: ${application.count}"${application.hovering ? '\n- AXStaticText "Hover information"' : ''}\n- [2] AXMenuBarItem "File"${params.includeMenus ? '\n - [3] AXMenuItem "Fixture command"' : '\nMenu contents omitted; request includeMenus=true to expand.'}`, elements: [{ elementId: '0', role: 'AXButton', title: 'Increment', value: null, description: null, diff --git a/snapshots/session/cua-ptc/replay.override.json b/snapshots/session/cua-ptc/replay.override.json index 63e4009990..fc162fc580 100644 --- a/snapshots/session/cua-ptc/replay.override.json +++ b/snapshots/session/cua-ptc/replay.override.json @@ -14,7 +14,7 @@ "type": "tool-call", "id": "cua-ptc-native-result", "name": "run_code", - "arguments": "{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst state = await tools.cua({ method: 'getState', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}" + "arguments": "{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst target = { appHandle: app.appHandle, windowId: 7 };\\nconst first = await tools.cua({ method: 'getState', params: { ...target, maxElements: 1 } });\\nconst state = first.truncated ? await tools.cua({ method: 'getState', params: { ...target, maxElements: 16 } }) : first;\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, initiallyTruncated: first.truncated, recoveredTextField: !first.tree.includes('AXTextField') && state.tree.includes('AXTextField'), hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}" } }, { From aeb278ce98ef9838db959fb1ea557068a8fbb118 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 14:24:58 +0800 Subject: [PATCH 08/10] test(cua): replay recovery from truncated AX observations --- packages/experimental/cua/src/index.ts | 2 +- .../cua-native/system-prompt.expected.md | 4 +++- .../cua-native/tool-schemas.expected.json | 2 ++ snapshots/session/cua-ptc/session.v4.jsonl | 18 ++++++++++-------- .../session/cua-ptc/system-prompt.expected.md | 4 +++- .../system-prompt.expected.md | 4 +++- 6 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index e1054b80ba..3c9a1cc6ef 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -155,7 +155,7 @@ export function installCua(ctx: Context, config: CuaConfig, backend: CuaBackend) const mouseControl = systemPointer ? new MouseControl(ctx, config.requireSystemPointerApproval) : undefined const requestSchema = systemPointer ? SystemPointerRequest : Request const instructions = systemPointer ? withSystemPointerGuidance(backend.guidance) : backend.guidance - const guidance = `${instructions}\n\nAX read defaults: maxElements=${config.maxElements}, maxDepth=${config.maxDepth}.\nMaximum click holdMs: ${config.maxClickHoldMs} ms.` + const guidance = `${instructions}\n\nAX read defaults: maxElements=${config.maxElements}, maxDepth=${config.maxDepth}.\n\nMaximum click holdMs: ${config.maxClickHoldMs} ms.` const native: NativeClientConfig = { helperPath, cwd: process.cwd(), timeoutMs: config.timeoutMs, axTimeoutMs: config.axTimeoutMs, connectTimeoutMs: config.connectTimeoutMs, screenshotTimeoutMs: config.screenshotTimeoutMs, diff --git a/snapshots/session/cua-native/system-prompt.expected.md b/snapshots/session/cua-native/system-prompt.expected.md index 8de11e7f91..213466834f 100644 --- a/snapshots/session/cua-native/system-prompt.expected.md +++ b/snapshots/session/cua-native/system-prompt.expected.md @@ -57,7 +57,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS accepts maxElements up to 100000 and maxDepth up to 512. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. @@ -85,4 +85,6 @@ Raw tools use { method, params }: include appHandle for app calls and windowId f Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. When approval is disabled an explicit move can act immediately. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. +AX read defaults: maxElements=5000, maxDepth=50. + Maximum click holdMs: 5000 ms. diff --git a/snapshots/session/cua-native/tool-schemas.expected.json b/snapshots/session/cua-native/tool-schemas.expected.json index eeeddd5e6f..68b762fd48 100644 --- a/snapshots/session/cua-native/tool-schemas.expected.json +++ b/snapshots/session/cua-native/tool-schemas.expected.json @@ -306,11 +306,13 @@ "type": "boolean" }, "maxElements": { + "description": "Maximum nodes to visit on this read. Omission uses the provider setting.", "type": "number", "minimum": 1, "multipleOf": 1 }, "maxDepth": { + "description": "Maximum depth to traverse on this read. Omission uses the provider setting.", "type": "number", "minimum": 1, "multipleOf": 1 diff --git a/snapshots/session/cua-ptc/session.v4.jsonl b/snapshots/session/cua-ptc/session.v4.jsonl index 87791413b2..3ed236e638 100644 --- a/snapshots/session/cua-ptc/session.v4.jsonl +++ b/snapshots/session/cua-ptc/session.v4.jsonl @@ -12,20 +12,22 @@ {"type":"request/header","data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"tools":"{{tools}}"},"reason":"initial"}} {"type":"request/context","data":{"provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"}} {"type":"session/title","data":{"title":"Use tools.cua from Node PTC","messageSeqs":[8],"source":{"kind":"fallback"}}} -{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-ptc-native-result","name":"run_code","arguments":"{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst state = await tools.cua({ method: 'getState', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791864698727,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1791864698727,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-ptc-native-result","name":"run_code","arguments":"{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst state = await tools.cua({ method: 'getState', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}"}}},{"type":"chunk","time":1791864698727,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791864698727,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} -{"type":"tool/call","data":{"turn":1,"step":1,"callId":"cua-ptc-native-result","name":"run_code","arguments":"{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst state = await tools.cua({ method: 'getState', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}"}} +{"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"cua-ptc-native-result","name":"run_code","arguments":"{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst target = { appHandle: app.appHandle, windowId: 7 };\\nconst first = await tools.cua({ method: 'getState', params: { ...target, maxElements: 1 } });\\nconst state = first.truncated ? await tools.cua({ method: 'getState', params: { ...target, maxElements: 16 } }) : first;\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, initiallyTruncated: first.truncated, recoveredTextField: !first.tree.includes('AXTextField') && state.tree.includes('AXTextField'), hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:4}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794205455434,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}},{"type":"chunk","time":1794205455435,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"cua-ptc-native-result","name":"run_code","arguments":"{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst target = { appHandle: app.appHandle, windowId: 7 };\\nconst first = await tools.cua({ method: 'getState', params: { ...target, maxElements: 1 } });\\nconst state = first.truncated ? await tools.cua({ method: 'getState', params: { ...target, maxElements: 16 } }) : first;\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, initiallyTruncated: first.truncated, recoveredTextField: !first.tree.includes('AXTextField') && state.tree.includes('AXTextField'), hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}"}}},{"type":"chunk","time":1794205455435,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794205455435,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}]},"surfaceOp":"append"} +{"type":"tool/call","data":{"turn":1,"step":1,"callId":"cua-ptc-native-result","name":"run_code","arguments":"{\"code\":\"const { Buffer } = await import('node:buffer');\\nconst app = await tools.cua({ method: 'getApp', params: { selector: 'test.fixture' } });\\nconst target = { appHandle: app.appHandle, windowId: 7 };\\nconst first = await tools.cua({ method: 'getState', params: { ...target, maxElements: 1 } });\\nconst state = first.truncated ? await tools.cua({ method: 'getState', params: { ...target, maxElements: 16 } }) : first;\\nconst shot = await tools.cua({ method: 'screenshot', params: { appHandle: app.appHandle, windowId: 7 } });\\nconst bytes = Buffer.from(shot.data, 'base64');\\nreturn { appHandle: app.appHandle, initiallyTruncated: first.truncated, recoveredTextField: !first.tree.includes('AXTextField') && state.tree.includes('AXTextField'), hasIncrement: state.tree.includes('Increment'), pngBytes: bytes.length, signature: bytes.subarray(0, 8).toString('hex'), imageWidth: shot.attachment.width, imageHeight: shot.attachment.height };\",\"description\":\"Read native CUA values and receive one screenshot\"}"}} {"type":"tool/ptc-dispatch-start","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:1","name":"cua","arguments":{"method":"getApp","params":{"selector":"test.fixture"}}}} {"type":"tool/ptc-dispatch","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:1","name":"cua","arguments":{"method":"getApp","params":{"selector":"test.fixture"}},"isError":false,"content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"name\":\"Fixture\",\"bundleId\":\"test.fixture\",\"running\":true}"}]}} -{"type":"tool/ptc-dispatch-start","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:2","name":"cua","arguments":{"method":"getState","params":{"appHandle":"fixture-app","windowId":7}}}} -{"type":"tool/ptc-dispatch","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:2","name":"cua","arguments":{"method":"getState","params":{"appHandle":"fixture-app","windowId":7}},"isError":false,"content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-1\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 0\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}]}} -{"type":"tool/ptc-dispatch-start","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:3","name":"cua","arguments":{"method":"screenshot","params":{"appHandle":"fixture-app","windowId":7}}}} -{"type":"tool/ptc-dispatch","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:3","name":"cua","arguments":{"method":"screenshot","params":{"appHandle":"fixture-app","windowId":7}},"isError":false,"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}]}} -{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-ptc-native-result"},"toolCallId":"cua-ptc-native-result","content":[{"type":"text","text":"{\n \"appHandle\": \"fixture-app\",\n \"hasIncrement\": true,\n \"pngBytes\": 90,\n \"signature\": \"89504e470d0a1a0a\",\n \"imageWidth\": 1,\n \"imageHeight\": 1\n}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} +{"type":"tool/ptc-dispatch-start","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:2","name":"cua","arguments":{"method":"getState","params":{"appHandle":"fixture-app","windowId":7,"maxElements":1}}}} +{"type":"tool/ptc-dispatch","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:2","name":"cua","arguments":{"method":"getState","params":{"appHandle":"fixture-app","windowId":7,"maxElements":1}},"isError":false,"content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-1\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\nAccessibility tree truncated (maxElements=1).\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":true}"}]}} +{"type":"tool/ptc-dispatch-start","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:3","name":"cua","arguments":{"method":"getState","params":{"appHandle":"fixture-app","windowId":7,"maxElements":16}}}} +{"type":"tool/ptc-dispatch","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:3","name":"cua","arguments":{"method":"getState","params":{"appHandle":"fixture-app","windowId":7,"maxElements":16}},"isError":false,"content":[{"type":"text","text":"{\"appHandle\":\"fixture-app\",\"windowId\":7,\"snapshotId\":\"fixture-snapshot-2\",\"tree\":\"AX scope=window offsets=UTF-16\\n- [0] AXButton \\\"Increment\\\"\\n- [1] AXTextField valueWritable=true selectionWritable=true value=\\\"\\\" placeholder=\\\"Fixture text\\\"\\n- AXStaticText \\\"Count: 0\\\"\\n- [2] AXMenuBarItem \\\"File\\\"\\nMenu contents omitted; request includeMenus=true to expand.\",\"windows\":[{\"windowId\":7,\"appName\":\"Fixture\",\"title\":\"CUA fixture\",\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"isOnScreen\":true,\"onCurrentSpace\":true}],\"truncated\":false}"}]}} +{"type":"tool/ptc-dispatch-start","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:4","name":"cua","arguments":{"method":"screenshot","params":{"appHandle":"fixture-app","windowId":7}}}} +{"type":"tool/ptc-dispatch","data":{"rootCallId":"cua-ptc-native-result","parentCallId":"cua-ptc-native-result","subCallId":"cua-ptc-native-result:ptc:4","name":"cua","arguments":{"method":"screenshot","params":{"appHandle":"fixture-app","windowId":7}},"isError":false,"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}]}} +{"type":"tool/result","data":{"turn":1,"step":1,"message":{"role":"tool","source":{"kind":"tool","callId":"cua-ptc-native-result"},"toolCallId":"cua-ptc-native-result","content":[{"type":"text","text":"{\n \"appHandle\": \"fixture-app\",\n \"initiallyTruncated\": true,\n \"recoveredTextField\": true,\n \"hasIncrement\": true,\n \"pngBytes\": 90,\n \"signature\": \"89504e470d0a1a0a\",\n \"imageWidth\": 1,\n \"imageHeight\": 1\n}"}],"isError":false,"id":"{{message:5}}"}},"sourceEventSeqs":[14],"surfaceOp":"append"} {"type":"agent/inbox/spliced","data":{"target":"next-step","start":0,"inserted":[{"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"source":{"kind":"ptc-mode"},"role":"user","id":"{{message:6}}"}]}} {"type":"step/end","data":{"turn":1,"step":1}} {"type":"agent/inbox/spliced","data":{"target":"next-step","start":0,"removedCount":1,"inserted":[]}} {"type":"step/start","data":{"turn":1,"step":2}} {"type":"user/message","data":{"content":[{"type":"text","text":"{\"pixelWidth\":1,\"pixelHeight\":1,\"windowId\":7,\"bounds\":{\"x\":40,\"y\":60,\"width\":320,\"height\":200},\"coordinateSpace\":\"screenshot-pixels\"}"},{"type":"image","attachment":{"attachmentId":"sha256:999f1d1527ee7e79266f16add5430fff76b1225d742464a5b1ff1f02971bb8ee","mediaType":"image/png","width":1,"height":1,"bytes":90}}],"source":{"kind":"ptc-mode"},"role":"user","id":"{{message:6}}"},"surfaceOp":"append"} -{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"Received native CUA values and one PNG screenshot through PTC. The PNG is 90 bytes with signature 89504e470d0a1a0a."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:7}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1791864699463,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"chunk","time":1791864699463,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"Received native CUA values and one PNG screenshot through PTC. The PNG is 90 bytes with signature 89504e470d0a1a0a."}}},{"type":"chunk","time":1791864699463,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1791864699463,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"Received native CUA values and one PNG screenshot through PTC. The PNG is 90 bytes with signature 89504e470d0a1a0a."}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"{{message:7}}"},"usage":{"inputTokens":3,"outputTokens":3},"stream":[{"type":"chunk","time":1794205456254,"chunk":{"type":"block-start","index":0,"blockType":"text"}},{"type":"chunk","time":1794205456254,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"Received native CUA values and one PNG screenshot through PTC. The PNG is 90 bytes with signature 89504e470d0a1a0a."}}},{"type":"chunk","time":1794205456254,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}},{"type":"chunk","time":1794205456254,"chunk":{"type":"finish","reason":{"kind":"stop"}}}]},"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":2}} {"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/snapshots/session/cua-ptc/system-prompt.expected.md b/snapshots/session/cua-ptc/system-prompt.expected.md index c024d5eb82..73cf79f81c 100644 --- a/snapshots/session/cua-ptc/system-prompt.expected.md +++ b/snapshots/session/cua-ptc/system-prompt.expected.md @@ -59,7 +59,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS accepts maxElements up to 100000 and maxDepth up to 512. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. @@ -87,6 +87,8 @@ Raw tools use { method, params }: include appHandle for app calls and windowId f Only if the task cannot be completed without moving the user’s pointer, call await win.mouseMove(target, { useSystemPointer: true }). Prefer an advertised accessibility action, a click on a visible control, or browser tools when possible. The question interrupts the user. When approval is disabled an explicit move can act immediately. Pass true on every physical move, even after approval. Omitting the option or passing false uses the default hover route without prompting. Approval or refusal lasts until this session closes or DSH restarts. A refusal sends no input. If the user dismisses the request, ask again only if they request it. Wait for an outstanding request to finish. Subagents cannot request physical movement. +AX read defaults: maxElements=5000, maxDepth=50. + Maximum click holdMs: 5000 ms. ## Writing code for run_code diff --git a/snapshots/session/ptc-script-retained/system-prompt.expected.md b/snapshots/session/ptc-script-retained/system-prompt.expected.md index 94583f8ff7..c658412356 100644 --- a/snapshots/session/ptc-script-retained/system-prompt.expected.md +++ b/snapshots/session/ptc-script-retained/system-prompt.expected.md @@ -59,7 +59,7 @@ Setup and discovery: Available at app or window level depending on the platform. Element ids are opaque strings from the latest successful tree; the object facade keeps one current snapshot per app across its repeated app and window objects, a new read replaces it, and a failed read invalidates the previous one. Repeated strings can appear for different elements, so do not reuse an old id blindly, and element operations do not fall back to pointer input or typing. -- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. On macOS, maxElements defaults to 5000 (up to 100000) and maxDepth defaults to 50 (up to 512). If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. +- getAccessibilityTree(options?: { emit?: boolean, includeMenus?: boolean, details?: boolean, maxElements?: number, maxDepth?: number }) -> string — set the node count or depth for one read. Omitted limits use the provider settings. includeMenus expands menu contents and details adds identifiers, native bounds, and complete action names. macOS accepts maxElements up to 100000 and maxDepth up to 512. If the tree is truncated, select a narrower window or raise the reported limit on the next read. In persistent PTC, capture a large tree with const tree = await win.getAccessibilityTree({ emit: false }). Filter its lines in memory, or save it with await tools.write({ file_path: path, content: tree }) to a new workspace file. Search its contents with tools.grep({ path, pattern }) and use tools.read around a matching line. Print only the matches you need. Use element ids from the file only while that observation is current. - listActions(elementId: string) -> string[] — list the actions an element advertises. - invoke(elementId: string) -> ActionResult — invoke the element's default action; dispatched is false when it has none. - performAction(elementId: string, action: string, argument?: JsonValue) -> ActionResult — perform an observed action with an optional direct JSON argument. @@ -85,6 +85,8 @@ Available at app or window level depending on the platform. Raw tools use { method, params }: include appHandle for app calls and windowId for window calls, and use snapshotId with elementId for element routes. The emit option is facade-only: emit is local output control, so raw callers must follow the raw operation schema for admitted params. getAccessibilityTree and getWindow read through raw getState; getWindows maps to listWindows; getScreenshot maps to screenshot; cua.getState maps to listApps; every other named operation uses the same method name. +AX read defaults: maxElements=2000, maxDepth=25. + Maximum click holdMs: 5000 ms. Persistent PTC initialization: From 9f5f2f5ae41e1f9a104eded05f9f2868f32775e2 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 16:09:18 +0800 Subject: [PATCH 09/10] test(cua): verify scoped provider guidance independently of read defaults --- packages/experimental/cua/tests/agent-scopes.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/experimental/cua/tests/agent-scopes.spec.ts b/packages/experimental/cua/tests/agent-scopes.spec.ts index 9404dd7af1..299e48397f 100644 --- a/packages/experimental/cua/tests/agent-scopes.spec.ts +++ b/packages/experimental/cua/tests/agent-scopes.spec.ts @@ -58,7 +58,7 @@ it('adds and removes CUA for existing and new PTC Agents without changing other await expect.poll(() => ctx.tools.get('cua', existing.agent)).toBeDefined() expect(ctx.tools.get('cua', standard.agent)).toBeUndefined() expect((await ctx.systemPrompt.assemble({ scope: existing.agent })).sections) - .toContainEqual(expect.objectContaining({ name: 'computer-use:dsh-cua', text: 'Fixture CUA guidance\n\nMaximum click holdMs: 5000 ms.' })) + .toContainEqual(expect.objectContaining({ name: 'computer-use:dsh-cua', text: expect.stringContaining('Fixture CUA guidance') })) await ctx.agentPresets.select(existing.agent, 'ptc') expect(ctx.tools.schemas(existing.agent).filter(tool => tool.name === 'cua')).toHaveLength(1) const fresh = await create('fresh', 'ptc') From 1fb39a2283aa876487d5d63b23c9d63ef90c3a67 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Mon, 9 Nov 2026 16:13:35 +0800 Subject: [PATCH 10/10] test(cua): inspect the typed prompt section directly --- packages/experimental/cua/tests/agent-scopes.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/experimental/cua/tests/agent-scopes.spec.ts b/packages/experimental/cua/tests/agent-scopes.spec.ts index 299e48397f..d1872e1e59 100644 --- a/packages/experimental/cua/tests/agent-scopes.spec.ts +++ b/packages/experimental/cua/tests/agent-scopes.spec.ts @@ -57,8 +57,8 @@ it('adds and removes CUA for existing and new PTC Agents without changing other const mounted = await ctx.plugin(provider, {}) await expect.poll(() => ctx.tools.get('cua', existing.agent)).toBeDefined() expect(ctx.tools.get('cua', standard.agent)).toBeUndefined() - expect((await ctx.systemPrompt.assemble({ scope: existing.agent })).sections) - .toContainEqual(expect.objectContaining({ name: 'computer-use:dsh-cua', text: expect.stringContaining('Fixture CUA guidance') })) + const { sections } = await ctx.systemPrompt.assemble({ scope: existing.agent }) + expect(sections.find(section => section.name === 'computer-use:dsh-cua')?.text).toContain('Fixture CUA guidance') await ctx.agentPresets.select(existing.agent, 'ptc') expect(ctx.tools.schemas(existing.agent).filter(tool => tool.name === 'cua')).toHaveLength(1) const fresh = await create('fresh', 'ptc')