diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 0a04181815..15f0be12d5 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -956,6 +956,8 @@ export interface Config { helperPath: string /** Deadline for one native operation, including capture and AX traversal. */ timeoutMs: number + /** Deadline for each individual Accessibility system call. */ + axTimeoutMs: number /** Maximum bytes retained for one helper response frame. */ maxResponseBytes: number /** Maximum UTF-8 bytes admitted for one helper request frame. */ diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 027a76b4f1..f22d4dc09d 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -958,6 +958,8 @@ export interface Config { helperPath: string /** Deadline for one native operation, including capture and AX traversal. */ timeoutMs: number + /** Deadline for each individual Accessibility system call. */ + axTimeoutMs: number /** Maximum bytes retained for one helper response frame. */ maxResponseBytes: number /** Maximum UTF-8 bytes admitted for one helper request frame. */ diff --git a/packages/experimental/cua/README.i18n.yaml b/packages/experimental/cua/README.i18n.yaml index a3e4374a21..069b5bee8b 100644 --- a/packages/experimental/cua/README.i18n.yaml +++ b/packages/experimental/cua/README.i18n.yaml @@ -15,8 +15,8 @@ en: d152484eb41ac6b4 zh: 7b30b973a849247d /deepseek-ai-dsh-experimental-cua/use-this-package: - en: 9d52a31412b64c77 - zh: 35a9f70cc50778a4 + en: caff9cfbb38fddc8 + zh: cf470484c6133654 /deepseek-ai-dsh-experimental-cua/use-this-package/connect-an-external-execution-environment: en: dc2f1cd930a6b6ba zh: 12b4f57fa745afbf @@ -45,8 +45,8 @@ en: a2c9acb5a4320dda zh: e8d1586af60215c4 /deepseek-ai-dsh-experimental-cua/known-limitations-and-deferred-work: - en: 0bc15ef067ffd2c6 - zh: 5ffd97b61a8a589f + en: 0fd701796488ac03 + zh: 9ce1ed68bc3cd624 /deepseek-ai-dsh-experimental-cua/known-limitations-and-deferred-work/dev-note: en: b836f01bd0bceb4a zh: 8122fa08b2aa95fa diff --git a/packages/experimental/cua/README.md b/packages/experimental/cua/README.md index 1b745041bb..0267bb2e17 100644 --- a/packages/experimental/cua/README.md +++ b/packages/experimental/cua/README.md @@ -40,6 +40,7 @@ The dump command checks profile composition without a model request. For a real | --- | --- | --- | | `helperPath` | required | Absolute native executable path. | | `timeoutMs` | `30000` | Per-operation deadline; cancellation can leave partial input. | +| `axTimeoutMs` | `3000` | Deadline per Accessibility system call, configured independently of the complete operation. | | `maxRequestBytes` / `maxResponseBytes` | `1048576` / `33554432` | UTF-8 request and response limits. | | `maxElements` / `maxDepth` | `2000` / `25` | Accessibility traversal limits. | | `dragDurationMs` / `dragSteps` | `400` / `20` | Duration and movement segments for a drag. | @@ -145,7 +146,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a hidden window, another Space or Stage Manager prevents usable observation, explain the foreground-dependent step before app.activate() or an advertised raise action, then read fresh state. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, initialize once with var { connectCua } = await import("@deepseek-ai/dsh-experimental-cua/bridge"); var cua = connectCua(() => tools.cua, text => console.log(text)). Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value. ``` @@ -165,6 +166,8 @@ Tool results append to Session history. Changing plugin configuration or tool pr - Only the macOS native implementation is provided. Browser tabs, app launching, rich-text clipboard formats, AX diffs, unassociated system panels and accessory-only apps are unsupported. - Coordinates use window-local points including the title bar. PNG pixel dimensions can differ. Window movement, application changes and other desktop users can race observations and input. - macOS Stage Manager can reduce background window bounds and screenshots to thumbnails. Full-window observation and coordinate input in this state are not guaranteed; model guidance requires explaining foreground needs and avoiding repeated focus reclamation. +- Background input depends on private SkyLight symbols; macOS updates can make them unavailable. Validate each target OS version. +- Observed failures include dispatched Unicode text not appearing in Blender and background Command+A not selecting all text in AppKit. Inspect application state after input; a successful tool result confirms dispatch only. - Native binaries require a local build. Signing, distribution and permission onboarding remain future work. - Persistent PTC is an external dependency assumption, not an implementation in this package. Real-model task success requires separate evaluation beyond native and protocol fixtures. diff --git a/packages/experimental/cua/README.zh.md b/packages/experimental/cua/README.zh.md index 7a405058d7..40a57a9c91 100644 --- a/packages/experimental/cua/README.zh.md +++ b/packages/experimental/cua/README.zh.md @@ -40,6 +40,7 @@ pnpm dsh headless --patch packages/experimental/cua/examples/macos.cordis.yml -- | --- | --- | --- | | `helperPath` | 必填 | 原生可执行文件的绝对路径。 | | `timeoutMs` | `30000` | 单次操作截止时间，取消后可能已发生部分输入。 | +| `axTimeoutMs` | `3000` | 每次 Accessibility 系统调用的截止时间，与完整操作截止独立配置。 | | `maxRequestBytes` / `maxResponseBytes` | `1048576` / `33554432` | UTF-8 请求与响应字节限制。 | | `maxElements` / `maxDepth` | `2000` / `25` | 辅助功能树遍历限制。 | | `dragDurationMs` / `dragSteps` | `400` / `20` | 拖动时长和移动分段数。 | @@ -145,7 +146,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a hidden window, another Space or Stage Manager prevents usable observation, explain the foreground-dependent step before app.activate() or an advertised raise action, then read fresh state. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, initialize once with var { connectCua } = await import("@deepseek-ai/dsh-experimental-cua/bridge"); var cua = connectCua(() => tools.cua, text => console.log(text)). Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value. ``` @@ -165,6 +166,8 @@ Window objects expose observation and pointer/AX actions; keyboard input remains - 只提供 macOS 原生实现。不支持浏览器标签、启动应用、富文本剪贴板格式、AX 差分、无法确认应用归属的系统面板，以及仅后台辅助类型的应用。 - 坐标使用包含标题栏的窗口内部 point，可能不同于 PNG 像素尺寸。窗口移动、应用变化及其他桌面使用者的操作可能与观察和输入交错。 - macOS 台前调度可能将后台窗口的尺寸与截图变为缩略图，当前无法保证这种状态下的完整画面与坐标输入；模型指引要求说明前台需求并避免反复抢回焦点。 +- 后台输入依赖私有 SkyLight 符号，macOS 升级可能使其不可用，需在目标系统版本验证。 +- 实测 Blender Unicode 输入曾出现分派成功但界面未显示文本，后台 AppKit 的 Command+A 曾未选中全文。输入后必须检查应用状态；工具返回成功只表示已分派。 - 原生二进制需要本地构建。签名、分发和权限引导尚未实现。 - 持久 PTC 是对外部运行环境的假设，本包不实现它。真实模型任务成功率需要在原生测试和协议测试之外单独评估。 diff --git a/packages/experimental/cua/native/README.txt b/packages/experimental/cua/native/README.txt index 3a2e480b38..c003c8ced6 100644 --- a/packages/experimental/cua/native/README.txt +++ b/packages/experimental/cua/native/README.txt @@ -6,17 +6,17 @@ Build with `swift build --package-path packages/experimental/cua/native --config Each stdin line is `{ "id": "opaque request id", "method": "method name", "params": {} }`. Each stdout line contains the same ID and exactly one of `result` or `error: {code, message}`. Malformed envelopes return `INVALID_REQUEST` with a null ID. Requests execute serially. Diagnostics must not be written to stdout. `protocol.schema.json` defines request fields. -AppKit runs on the main event loop. Blocking AX calls and synchronous input use an actor on a dedicated serial executor; standard-input reading and response encoding/writing use separate queues. The coordinator admits one request through completion and output. The host passes its byte limits, operation deadline and shutdown grace through `--max-request-bytes`, `--max-response-bytes`, `--ax-timeout-ms` and `--shutdown-grace-ms`. Unicode text event spacing is set by `--text-event-interval-ms` (0..1000 ms, default 0.25); it does not change chord timing. The AX timeout bounds each individual system call; the host still owns the complete operation deadline. Cancellation stops further input between events and releases only this operation’s held keys/buttons when possible; forced process termination cannot guarantee release. +AppKit runs on the main event loop. Blocking AX calls and synchronous input use an actor on a dedicated serial executor; standard-input reading and response encoding/writing use separate queues. The coordinator admits one request through completion and output. The host passes its byte limits, per-call AX deadline and shutdown grace through `--max-request-bytes`, `--max-response-bytes`, `--ax-timeout-ms` and `--shutdown-grace-ms`. Unicode text event spacing is set by `--text-event-interval-ms` (0..1000 ms, default 0.25); it does not change chord timing. The provider configures AX timeout separately as axTimeoutMs (default 3000 ms). The AX timeout bounds each individual system call; the host still owns the complete operation deadline. Cancellation stops further input between events and releases only this operation’s held keys/buttons when possible; forced process termination cannot guarantee release. Successful `activate` and `performSecondaryAction` AXRaise results include `foreground` alongside `{dispatched: true, verified: false}`. It records `requestId`, `operation`, `targetPid`, nullable `targetWindowId`, `activationRequested`, `raiseRequested`, and immediate `before`/`after` observations. `frontPid` comes from WindowServer; `focusedWindowId` is the target app's AX-focused window; `frontWindowId` and `frontWindowPid` identify the first visible normal window. Unknown values are null. Observations are sequential, not atomic or proof of causality. Tool-owned requests use the logged tool call ID. Failed operations retain the existing error response and may have partial effects. | Method | Result | | --- | --- | -| `getCapabilities` | `{platform: "macos", permissions: {accessibility, screenRecording}, methods, coordinateSpace: "window-points", browser: false}`. Permission values are boolean preflight observations; no system prompt opens. | +| `getCapabilities` | `{platform: "macos", permissions: {accessibility, screenRecording}, sessionLocked, methods, coordinateSpace: "window-points", browser: false}`. Permission values are boolean preflight observations; no system prompt opens. | | `listApps` | Array of `{pid, name, bundleId}` for running regular applications. | | `getApp` | `{appHandle, pid, name, bundleId}` for one running app selected by exact name, bundle ID, or decimal PID. Ambiguous names fail. | | `listWindows` | Owned normal windows with `{windowId, pid, appName, title, bounds, isOnScreen, onCurrentSpace}`. Space membership can be null. | -| `getState` | `{appHandle, pid, windowId, snapshotId, tree, elements, windows, truncated}`. `windowId` is null for an app-wide observation. Elements include index, role, title, value, description, identifier, bounds, actions, and enabled state. Element bounds are null or [x, y, width, height] in screen points; window input uses local points. These structured element diagnostics are retained by native checks and omitted by the model tool projection. Unavailable optional descriptive properties are null, and an unavailable action list is empty; permission, communication and window-discovery failures still reject the observation. | +| `getState` | `{appHandle, pid, windowId, focusedWindowId, snapshotId, tree, elements, windows, truncated}`. `windowId` is null for an app-wide observation. Elements include index, role, title, value, description, identifier, bounds, actions, and enabled state. Element bounds are null or [x, y, width, height] in screen points; window input uses local points. These structured element diagnostics are retained by native checks and omitted by the model tool projection. Unavailable optional descriptive properties are null, and an unavailable action list is empty; permission, communication and window-discovery failures still reject the observation. | | `click`, `setValue`, `typeText`, `pressKey`, `scroll`, `drag` | `{dispatched: true, verified: false}` records native dispatch. Read fresh state to verify the intended application change. | | `screenshot` | `{mimeType: "image/png", data, pixelWidth, pixelHeight, bounds, coordinateSpace: "window-points", windowId}`. PNG pixel dimensions can differ from window point dimensions on Retina displays. | @@ -53,3 +53,5 @@ For text pacing comparisons, use `native/tests/text_input_benchmark.py` with `-- For an independently isolated Electron/Chromium check, run `python3 packages/experimental/cua/native/tests/electron_e2e.py --electron /absolute/path/to/Electron.app/Contents/MacOS/Electron`. Omitting `--electron` uses an already installed desktop workspace runtime. The check never downloads Electron. Its renderer runs with Node integration disabled, context isolation and Chromium sandboxing enabled, private userData/sessionData/log directories, and a local fixture page. Renderer click/input events write independent observations through the main process; all test actions still use the native helper. AX polling waits for actual controls after Chromium accessibility activation. Native and Electron subprocess groups are owned and stopped before temporary files are removed. Optional artifacts include a fixture-only screenshot and private request transcript. System file-panel keyboard routing resolves the remote recipient through the selected app’s focused AX sheet. Only Apple’s Open and Save Panel Service is accepted, and its focused ancestry must still contain the identified remote window. Element pointer fallbacks resolve an app-owned window or sheet through AX parents when AXWindow is unavailable. AX actions retain priority over pointer input; visual click markers need no system-cursor movement. + +Keyboard modifiers in this native request schema accept command/control/option/shift/fn and cmd/ctrl/alt aliases. The TypeScript raw-tool schema accepts only canonical names; the object API normalizes key-chord aliases before dispatch. diff --git a/packages/experimental/cua/src/index.ts b/packages/experimental/cua/src/index.ts index b01781a0de..757c638f6a 100644 --- a/packages/experimental/cua/src/index.ts +++ b/packages/experimental/cua/src/index.ts @@ -24,6 +24,8 @@ export interface Config { helperPath: string /** Deadline for one native operation, including capture and AX traversal. */ timeoutMs: number + /** Deadline for each individual Accessibility system call. */ + axTimeoutMs: number /** Maximum bytes retained for one helper response frame. */ maxResponseBytes: number /** Maximum UTF-8 bytes admitted for one helper request frame. */ @@ -46,6 +48,7 @@ export interface Config { export const Config: Schema = Schema.object({ helperPath: Schema.string().required(), timeoutMs: Schema.number().step(1).min(1).max(2 ** 31 - 1).default(30_000), + axTimeoutMs: Schema.number().step(1).min(1).max(2 ** 31 - 1).default(3000), maxRequestBytes: Schema.number().step(1).min(1024).default(1024 * 1024), dragDurationMs: Schema.number().step(1).min(1).max(5000).default(400), dragSteps: Schema.number().step(1).min(1).max(240).default(20), @@ -85,7 +88,7 @@ App and Window methods: Raw operation tools use { method, params }, including params: {} for parameterless methods. getApp selects without activating the app. Keep applications in the background for observation and routine input. Do not activate or raise an app during initialization, before each action, or merely because another app is foreground. Prefer advertised AX actions. If a hidden window, another Space or Stage Manager prevents usable observation, explain the foreground-dependent step before app.activate() or an advertised raise action, then read fresh state. Avoid repeated activation when the user switches away; continue background-capable steps and report what requires foreground access. Applications may still change focus in response to input. -Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission. +Await operations sequentially. After input, read fresh state to verify the effect. A successful action means dispatched, not verified completion. Cancellation may leave partial effects; never retry uncertain input without observing. STALE_APP requires getApp again; STALE_SNAPSHOT requires getState again. WINDOW_CHANGED requires fresh state before any retry; use an advertised AX action or explicitly restore the window before screenshot or coordinate input, then observe again. PERMISSION_DENIED requires the user to grant macOS Accessibility or Screen Recording permission. Window objects expose observation and pointer/AX actions; keyboard input remains app-scoped and follows current focus. Scroll deltas are integer lines from -10 to 10; positive values reveal content above or left. Paste inserts plain text using native Unicode input without changing the clipboard; rich-text formats are unavailable. Browser applications can be controlled with getApp like other native apps; browser tabs and app launching are unavailable. The execution host must install the object API and own variable persistence; this plugin does not preload a cua global. In persistent PTC, initialize once with var { connectCua } = await import("@deepseek-ai/dsh-experimental-cua/bridge"); var cua = connectCua(() => tools.cua, text => console.log(text)). Retain app objects between calls. End screenshot calls with void await app.getScreenshot() or void await app.getAXStateAndScreenshot() so image bytes are not serialized as the program completion value.` @@ -155,7 +158,7 @@ export function apply(ctx: Context, config: Config): void { let client = clients.get(agent) if (!client || client.disposed) { client = new NativeClient(ctx, { - helperPath: config.helperPath, cwd: process.cwd(), timeoutMs: config.timeoutMs, + helperPath: config.helperPath, cwd: process.cwd(), timeoutMs: config.timeoutMs, axTimeoutMs: config.axTimeoutMs, maxRequestBytes: config.maxRequestBytes, maxResponseBytes: config.maxResponseBytes, graceMs: config.graceMs, textEventIntervalMs: config.textEventIntervalMs, }) diff --git a/packages/experimental/cua/src/native-client.ts b/packages/experimental/cua/src/native-client.ts index 726a8c17c4..c0c0fff7c6 100644 --- a/packages/experimental/cua/src/native-client.ts +++ b/packages/experimental/cua/src/native-client.ts @@ -28,6 +28,7 @@ export interface NativeClientConfig { helperPath: string cwd: string timeoutMs: number + axTimeoutMs: number maxRequestBytes: number maxResponseBytes: number graceMs: number @@ -51,7 +52,7 @@ export class NativeClient { config.helperPath, '--max-request-bytes', String(config.maxRequestBytes), '--max-response-bytes', String(config.maxResponseBytes), - '--ax-timeout-ms', String(config.timeoutMs), + '--ax-timeout-ms', String(config.axTimeoutMs), '--shutdown-grace-ms', String(config.graceMs), '--text-event-interval-ms', String(config.textEventIntervalMs), ], cwd: config.cwd, graceMs: config.graceMs, diff --git a/packages/experimental/cua/tests/native-client.spec.ts b/packages/experimental/cua/tests/native-client.spec.ts index 07f4f2b373..940c2d0d75 100644 --- a/packages/experimental/cua/tests/native-client.spec.ts +++ b/packages/experimental/cua/tests/native-client.spec.ts @@ -35,7 +35,7 @@ function fixture(options: { } const spawn = vi.fn(() => handle) const client = new NativeClient({ subprocess: { spawn } }, { - helperPath: '/fixture/helper', cwd: '/fixture', timeoutMs: options.timeoutMs ?? 60_000, + helperPath: '/fixture/helper', cwd: '/fixture', axTimeoutMs: 321, timeoutMs: options.timeoutMs ?? 60_000, maxRequestBytes: options.maxRequestBytes ?? 1024, maxResponseBytes: options.maxResponseBytes ?? 1024, graceMs: 10, textEventIntervalMs: 0.5, }) @@ -60,7 +60,7 @@ describe('native helper transport', () => { const f = fixture({ maxRequestBytes: 4096, maxResponseBytes: 8192, timeoutMs: 1234 }) expect(f.spawn).toHaveBeenCalledWith(expect.objectContaining({ argv: [ '/fixture/helper', '--max-request-bytes', '4096', '--max-response-bytes', '8192', - '--ax-timeout-ms', '1234', '--shutdown-grace-ms', '10', '--text-event-interval-ms', '0.5', + '--ax-timeout-ms', '321', '--shutdown-grace-ms', '10', '--text-event-interval-ms', '0.5', ] })) }) diff --git a/packages/experimental/cua/tests/provider-lifecycle.spec.ts b/packages/experimental/cua/tests/provider-lifecycle.spec.ts index 3ecdb1014e..349683d9f2 100644 --- a/packages/experimental/cua/tests/provider-lifecycle.spec.ts +++ b/packages/experimental/cua/tests/provider-lifecycle.spec.ts @@ -27,7 +27,7 @@ vi.mock('../src/native-client.ts', async importOriginal => ({ })) const config = { - helperPath: resolve('fixture-helper'), timeoutMs: 10_000, maxRequestBytes: 1024, maxResponseBytes: 4096, + helperPath: resolve('fixture-helper'), timeoutMs: 10_000, axTimeoutMs: 3000, maxRequestBytes: 1024, maxResponseBytes: 4096, graceMs: 10, textEventIntervalMs: 8, maxElements: 64, maxDepth: 8, dragDurationMs: 200, dragSteps: 10, } satisfies NativeProvider.Config